kylin git commit: minor, add util PrintHBaseConfig

[liyang]

Repository: kylin
Updated Branches:
  refs/heads/master 93e30208d -> b80762cd1


minor, add util PrintHBaseConfig


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b80762cd
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b80762cd
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b80762cd

Branch: refs/heads/master
Commit: b80762cd1dcc410179ab366391588486e1028ffa
Parents: 93e3020
Author: Li Yang 
Authored: Thu Nov 3 13:50:44 2016 +0800
Committer: Li Yang 
Committed: Thu Nov 3 13:51:42 2016 +0800

--
 .../storage/hbase/util/PrintHBaseConfig.java| 62 
 1 file changed, 62 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/b80762cd/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PrintHBaseConfig.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PrintHBaseConfig.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PrintHBaseConfig.java
new file mode 100644
index 000..634ebdf
--- /dev/null
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PrintHBaseConfig.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.storage.hbase.util;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.Properties;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+
+/**
+ */
+public class PrintHBaseConfig {
+
+public static void main(String[] args) throws IOException {
+MyConfig config = new MyConfig(HBaseConfiguration.create());
+
+if (args.length == 0) {
+for (Map.Entry item: config.getProps().entrySet()) {
+System.out.println(item.getKey() + "=" + item.getValue());
+}
+System.exit(0);
+}
+
+if (args.length == 1) {
+System.out.println(config.get(args[0]));
+System.exit(0);
+}
+
+for (String arg : args) {
+System.out.println(arg + "=" + config.get(arg));
+}
+System.exit(0);
+}
+
+private static class MyConfig extends Configuration {
+MyConfig(Configuration other) {
+super(other);
+}
+
+protected synchronized Properties getProps() {
+return super.getProps();
+}
+}
+}

kylin git commit: KYLIN-2154 source table rows will be skipped if TOPN's group column contains NULL values

[shaofengshi]

Repository: kylin
Updated Branches:
  refs/heads/v1.6.0-rc1 df9a80869 -> 9a2fed5ba


KYLIN-2154 source table rows will be skipped if TOPN's group column contains 
NULL values


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9a2fed5b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9a2fed5b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9a2fed5b

Branch: refs/heads/v1.6.0-rc1
Commit: 9a2fed5bad2747f4570f92fd4f3f5380ca2c8ff4
Parents: df9a808
Author: Hongbin Ma 
Authored: Wed Nov 2 22:28:35 2016 +0800
Committer: shaofengshi 
Committed: Thu Nov 3 13:41:44 2016 +0800

--
 .../java/org/apache/kylin/measure/topn/TopNMeasureType.java | 9 +++--
 .../apache/kylin/storage/gtrecord/CubeSegmentScanner.java   | 5 -
 2 files changed, 11 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/9a2fed5b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
 
b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
index 7e788e4..f34c57c 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
@@ -18,6 +18,7 @@
 
 package org.apache.kylin.measure.topn;
 
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.Iterator;
 import java.util.List;
@@ -142,8 +143,12 @@ public class TopNMeasureType extends 
MeasureType {
 final ByteArray key = new ByteArray(keyLength);
 int offset = 0;
 for (int i = 0; i < dimensionEncodings.length; i++) {
-byte[] valueBytes = Bytes.toBytes(values[i + 1]);
-dimensionEncodings[i].encode(valueBytes, 
valueBytes.length, key.array(), offset);
+if (values[i + 1] == null) {
+Arrays.fill(key.array(), offset, offset + 
dimensionEncodings[i].getLengthOfEncoding(), DimensionEncoding.NULL);
+} else {
+byte[] valueBytes = Bytes.toBytes(values[i + 1]);
+dimensionEncodings[i].encode(valueBytes, 
valueBytes.length, key.array(), offset);
+}
 offset += dimensionEncodings[i].getLengthOfEncoding();
 }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/9a2fed5b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
--
diff --git 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
index 6e19c09..c6a6daa 100644
--- 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
+++ 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
@@ -53,6 +53,9 @@ public class CubeSegmentScanner implements IGTScanner {
 
 public CubeSegmentScanner(CubeSegment cubeSeg, Cuboid cuboid, 
Set dimensions, Set groups, //
 Collection metrics, TupleFilter originalfilter, 
StorageContext context, String gtStorage) {
+
+logger.info("Init CubeSegmentScanner for segment {}", 
cubeSeg.getName());
+
 this.cuboid = cuboid;
 this.cubeSeg = cubeSeg;
 
@@ -61,7 +64,7 @@ public class CubeSegmentScanner implements IGTScanner {
 //is working on its own copy
 byte[] serialize = TupleFilterSerializer.serialize(originalfilter, 
StringCodeSystem.INSTANCE);
 TupleFilter filter = TupleFilterSerializer.deserialize(serialize, 
StringCodeSystem.INSTANCE);
-
+
 // translate FunctionTupleFilter to IN clause
 ITupleFilterTransformer translator = new 
BuiltInFunctionTransformer(cubeSeg.getDimensionEncodingMap());
 filter = translator.transform(filter);

kylin git commit: KYLIN-2070 minor change in the ‘storage’ tab

[shaofengshi]

Repository: kylin
Updated Branches:
  refs/heads/master f9adf10d2 -> 93e30208d


KYLIN-2070 minor change in the ‘storage’ tab

Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/93e30208
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/93e30208
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/93e30208

Branch: refs/heads/master
Commit: 93e30208d4cf96e460948bc9df5050e43240e89b
Parents: f9adf10
Author: shaofengshi 
Authored: Tue Oct 25 13:03:12 2016 +0800
Committer: shaofengshi 
Committed: Thu Nov 3 13:40:16 2016 +0800

--
 webapp/app/partials/cubes/cube_detail.html | 27 +++--
 1 file changed, 12 insertions(+), 15 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/93e30208/webapp/app/partials/cubes/cube_detail.html
--
diff --git a/webapp/app/partials/cubes/cube_detail.html 
b/webapp/app/partials/cubes/cube_detail.html
index 909e05e..1896b42 100755
--- a/webapp/app/partials/cubes/cube_detail.html
+++ b/webapp/app/partials/cubes/cube_detail.html
@@ -38,7 +38,7 @@
 
 
-HBase
+Storage
 
 
 
@@ -95,27 +95,24 @@
 
   
 
+
+  
+  Segment Number: {{cube.hbase.length}} Total Size: {{cube.totalSize | bytes}}
+
 
-HTable: {{table.tableName}}
+Segment: {{table.segmentName}}
 
-Segment Name: {{table.segmentName}}
-Segment Status: {{table.segmentStatus}}
-Source Count: {{table.sourceCount}}
-SourceOffsetStart: {{table.sourceOffsetStart|reverseToGMT0}}
-SourceOffsetEnd: {{table.sourceOffsetEnd|reverseToGMT0}}
-Region Count: {{table.regionCount}}
-Size: {{table.tableSize | 
bytes}}
+Status: {{table.segmentStatus}}
 Start Time: {{table.dateRangeStart | 
reverseToGMT0}}
 End Time: {{table.dateRangeEnd | 
reverseToGMT0}}
+Source Count: {{table.sourceCount}}
+HBase Table: {{table.tableName}}
+Region Count: {{table.regionCount}}
+Size: {{table.tableSize | 
bytes}}
 
 
-
-
-Total Size: {{cube.totalSize | 
bytes}}
-Total Number: {{cube.hbase.length}}
-
 
-No HBase Info.
+No Storage Info.
 
 
 

[4/7] kylin git commit: bug fix in integer as partition column

[mahongbin]

bug fix in integer as partition column


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/99eb6488
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/99eb6488
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/99eb6488

Branch: refs/heads/master
Commit: 99eb6488227b795b63349851fabce302d7697082
Parents: d9c118b
Author: Hongbin Ma 
Authored: Tue Nov 1 09:40:46 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Nov 3 13:34:12 2016 +0800

--
 .../org/apache/kylin/cube/gridtable/SegmentGTStartAndEnd.java| 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/99eb6488/core-cube/src/main/java/org/apache/kylin/cube/gridtable/SegmentGTStartAndEnd.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/SegmentGTStartAndEnd.java
 
b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/SegmentGTStartAndEnd.java
index b4a82d4..8ceb841 100644
--- 
a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/SegmentGTStartAndEnd.java
+++ 
b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/SegmentGTStartAndEnd.java
@@ -24,10 +24,10 @@ import org.apache.commons.lang.StringUtils;
 import org.apache.kylin.common.util.ByteArray;
 import org.apache.kylin.common.util.DateFormat;
 import org.apache.kylin.common.util.Pair;
-import org.apache.kylin.metadata.model.ISegment;
 import org.apache.kylin.dimension.AbstractDateDimEnc;
 import org.apache.kylin.gridtable.GTInfo;
 import org.apache.kylin.metadata.datatype.DataType;
+import org.apache.kylin.metadata.model.ISegment;
 
 public class SegmentGTStartAndEnd {
 private ISegment segment;
@@ -67,7 +67,7 @@ public class SegmentGTStartAndEnd {
 value = DateFormat.formatToDateStr(ts);
 } else if (partitionColType.isTimeFamily()) {
 value = DateFormat.formatToTimeWithoutMilliStr(ts);
-} else if (partitionColType.isStringFamily()) {
+} else if (partitionColType.isStringFamily() || 
partitionColType.isIntegerFamily()) {//integer like 20160101
 String partitionDateFormat = 
segment.getModel().getPartitionDesc().getPartitionDateFormat();
 if (StringUtils.isEmpty(partitionDateFormat))
 partitionDateFormat = DateFormat.DEFAULT_DATE_PATTERN;

[2/7] kylin git commit: StorageCleanupJob: delete intermediate tables due to kylin.hive.keep.flat.table=true

[mahongbin]

StorageCleanupJob: delete intermediate tables due to 
kylin.hive.keep.flat.table=true


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2c8d7759
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2c8d7759
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2c8d7759

Branch: refs/heads/master
Commit: 2c8d7759e8662eb2a9e103a7b25948156e630e2a
Parents: cad3def
Author: Hongbin Ma 
Authored: Wed Nov 2 11:13:24 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Nov 3 13:34:12 2016 +0800

--
 .../apache/kylin/tool/StorageCleanupJob.java| 61 
 1 file changed, 49 insertions(+), 12 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/2c8d7759/tool/src/main/java/org/apache/kylin/tool/StorageCleanupJob.java
--
diff --git a/tool/src/main/java/org/apache/kylin/tool/StorageCleanupJob.java 
b/tool/src/main/java/org/apache/kylin/tool/StorageCleanupJob.java
index 56681af..2a2d1f3 100644
--- a/tool/src/main/java/org/apache/kylin/tool/StorageCleanupJob.java
+++ b/tool/src/main/java/org/apache/kylin/tool/StorageCleanupJob.java
@@ -21,6 +21,7 @@ package org.apache.kylin.tool;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Map;
 import java.util.concurrent.Callable;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
@@ -49,6 +50,7 @@ import org.apache.kylin.common.util.OptionsHelper;
 import org.apache.kylin.cube.CubeInstance;
 import org.apache.kylin.cube.CubeManager;
 import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.engine.mr.HadoopUtil;
 import org.apache.kylin.engine.mr.JobBuilderSupport;
 import org.apache.kylin.job.engine.JobEngineConfig;
 import org.apache.kylin.job.execution.AbstractExecutable;
@@ -63,6 +65,7 @@ import org.slf4j.LoggerFactory;
 
 import com.google.common.base.Predicate;
 import com.google.common.collect.Iterables;
+import com.google.common.collect.Maps;
 
 public class StorageCleanupJob extends AbstractApplication {
 
@@ -252,6 +255,7 @@ public class StorageCleanupJob extends AbstractApplication {
 
 private void cleanUnusedIntermediateHiveTable(Configuration conf) throws 
Exception {
 final KylinConfig config = KylinConfig.getInstanceFromEnv();
+JobEngineConfig engineConfig = new 
JobEngineConfig(KylinConfig.getInstanceFromEnv());
 final CliCommandExecutor cmdExec = config.getCliCommandExecutor();
 final int uuidLength = 36;
 final String preFix = "kylin_intermediate_";
@@ -269,6 +273,7 @@ public class StorageCleanupJob extends AbstractApplication {
 List allJobs = executableManager.getAllJobIds();
 List allHiveTablesNeedToBeDeleted = new ArrayList();
 List workingJobList = new ArrayList();
+Map segmentId2JobId = Maps.newHashMap();
 
 StringBuilder sb = new StringBuilder();
 for (String jobId : allJobs) {
@@ -278,6 +283,11 @@ public class StorageCleanupJob extends AbstractApplication 
{
 workingJobList.add(jobId);
 sb.append(jobId).append("(").append(state).append("), ");
 }
+
+String segmentId = getSegmentIdFromJobId(jobId);
+if (segmentId != null) {//some jobs are not cubing jobs 
+segmentId2JobId.put(segmentId, jobId);
+}
 }
 logger.info("Working jobIDs: " + workingJobList);
 
@@ -302,15 +312,15 @@ public class StorageCleanupJob extends 
AbstractApplication {
 if (UUId_PATTERN.matcher(uuid).matches()) {
 //Check whether it's a hive table in use
 if (isTableInUse(uuid, workingJobList)) {
-logger.info("Skip because not isTableInUse");
+logger.info("Skip deleting because the table is in 
use");
 isNeedDel = false;
 }
 } else {
-logger.info("Skip because not match pattern");
+logger.info("Skip deleting because not match pattern");
 isNeedDel = false;
 }
 } else {
-logger.info("Skip because length not qualified");
+logger.info("Skip deleting because length not qualified");
 isNeedDel = false;
 }
 
@@ -320,19 +330,41 @@ public class StorageCleanupJob extends 
AbstractApplication {
 }
 
 if (delete == true) {
-final String useDatabaseHql = "USE " + 
config.getHiveDatabaseForIntermediateTable() + ";";
-final HiveCmdBuilder hiveCmdBuilder = new HiveCmdBuilder();
-

[7/7] kylin git commit: KYLIN-2145 StorageCleanupJob will fail when beeline enabled

[mahongbin]

KYLIN-2145 StorageCleanupJob will fail when beeline enabled


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/cad3def9
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/cad3def9
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/cad3def9

Branch: refs/heads/master
Commit: cad3def9e21fcfc07767e3a1b72f5b1bf8cae3de
Parents: 7358a78
Author: Hongbin Ma 
Authored: Tue Nov 1 18:32:31 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Nov 3 13:34:12 2016 +0800

--
 tool/pom.xml|  4 ++
 .../apache/kylin/tool/StorageCleanupJob.java| 44 ++--
 2 files changed, 25 insertions(+), 23 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/cad3def9/tool/pom.xml
--
diff --git a/tool/pom.xml b/tool/pom.xml
index e530469..e3d7bfa 100644
--- a/tool/pom.xml
+++ b/tool/pom.xml
@@ -42,6 +42,10 @@
 org.apache.kylin
 kylin-source-kafka
 
+
+org.apache.kylin
+kylin-source-hive
+
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/cad3def9/tool/src/main/java/org/apache/kylin/tool/StorageCleanupJob.java
--
diff --git a/tool/src/main/java/org/apache/kylin/tool/StorageCleanupJob.java 
b/tool/src/main/java/org/apache/kylin/tool/StorageCleanupJob.java
index c1ff753..56681af 100644
--- a/tool/src/main/java/org/apache/kylin/tool/StorageCleanupJob.java
+++ b/tool/src/main/java/org/apache/kylin/tool/StorageCleanupJob.java
@@ -18,9 +18,7 @@
 
 package org.apache.kylin.tool;
 
-import java.io.BufferedReader;
 import java.io.IOException;
-import java.io.StringReader;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.concurrent.Callable;
@@ -31,6 +29,8 @@ import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 import java.util.regex.Pattern;
 
+import javax.annotation.Nullable;
+
 import org.apache.commons.cli.Option;
 import org.apache.commons.cli.OptionBuilder;
 import org.apache.commons.cli.Options;
@@ -46,7 +46,6 @@ import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.AbstractApplication;
 import org.apache.kylin.common.util.CliCommandExecutor;
 import org.apache.kylin.common.util.OptionsHelper;
-import org.apache.kylin.common.util.Pair;
 import org.apache.kylin.cube.CubeInstance;
 import org.apache.kylin.cube.CubeManager;
 import org.apache.kylin.cube.CubeSegment;
@@ -56,10 +55,15 @@ import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableState;
 import org.apache.kylin.job.manager.ExecutableManager;
 import org.apache.kylin.metadata.realization.IRealizationConstants;
-import org.apache.kylin.storage.hbase.util.HiveCmdBuilder;
+import org.apache.kylin.source.hive.HiveClientFactory;
+import org.apache.kylin.source.hive.HiveCmdBuilder;
+import org.apache.kylin.source.hive.IHiveClient;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.google.common.base.Predicate;
+import com.google.common.collect.Iterables;
+
 public class StorageCleanupJob extends AbstractApplication {
 
 @SuppressWarnings("static-access")
@@ -244,27 +248,24 @@ public class StorageCleanupJob extends 
AbstractApplication {
 }
 
System.out.println("---");
 }
-
 }
 
-private void cleanUnusedIntermediateHiveTable(Configuration conf) throws 
IOException {
+private void cleanUnusedIntermediateHiveTable(Configuration conf) throws 
Exception {
 final KylinConfig config = KylinConfig.getInstanceFromEnv();
 final CliCommandExecutor cmdExec = config.getCliCommandExecutor();
 final int uuidLength = 36;
 final String preFix = "kylin_intermediate_";
 final String uuidPattern = 
"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}";
 
-
-final String useDatabaseHql = "USE " + 
config.getHiveDatabaseForIntermediateTable() + ";";
-final HiveCmdBuilder hiveCmdBuilder = new HiveCmdBuilder();
-hiveCmdBuilder.addStatement(useDatabaseHql);
-hiveCmdBuilder.addStatement("show tables " + 
"\'kylin_intermediate_*\'" + "; ");
-
-Pair result = cmdExec.execute(hiveCmdBuilder.build());
+IHiveClient hiveClient = HiveClientFactory.getHiveClient();
+List hiveTableNames = 
hiveClient.getHiveTableNames(config.getHiveDatabaseForIntermediateTable());
+Iterable kylinIntermediates = Iterables.filter(hiveTableNames, 
new Predicate() {
+@Override
+public 

[6/7] kylin git commit: minor fix for QueryUtil

[mahongbin]

minor fix for QueryUtil


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e3e9ad3a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e3e9ad3a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e3e9ad3a

Branch: refs/heads/master
Commit: e3e9ad3a2b347eee71aa6b243a2561e6f94fd370
Parents: 2c8d775
Author: Hongbin Ma 
Authored: Wed Nov 2 15:29:56 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Nov 3 13:34:12 2016 +0800

--
 .../src/main/java/org/apache/kylin/rest/util/QueryUtil.java| 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/e3e9ad3a/server-base/src/main/java/org/apache/kylin/rest/util/QueryUtil.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/util/QueryUtil.java 
b/server-base/src/main/java/org/apache/kylin/rest/util/QueryUtil.java
index 66619fe..eb73e8f 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/util/QueryUtil.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/util/QueryUtil.java
@@ -94,7 +94,7 @@ public class QueryUtil {
 private static final String SM = "\\s+";
 private static final Pattern PTN_GROUP_BY = Pattern.compile(S1 + 
"GROUP" + SM + "BY" + S1, Pattern.CASE_INSENSITIVE);
 private static final Pattern PTN_HAVING_COUNT_GREATER_THAN_ZERO = 
Pattern.compile(S1 + "HAVING" + SM + "[(]?" + S0 + "COUNT" + S0 + "[(]" + S0 + 
"1" + S0 + "[)]" + S0 + ">" + S0 + "0" + S0 + "[)]?", Pattern.CASE_INSENSITIVE);
-private static final Pattern PTN_SUM_1 = Pattern.compile(S1 + "SUM" + 
S0 + "[(]" + S0 + "[1]" + S0 + "[)]" + S1, Pattern.CASE_INSENSITIVE);
+private static final Pattern PTN_SUM_1 = Pattern.compile(S0 + "SUM" + 
S0 + "[(]" + S0 + "[1]" + S0 + "[)]" + S0, Pattern.CASE_INSENSITIVE);
 private static final Pattern PTN_INTERVAL = Pattern.compile("interval" 
+ SM + "(floor\\()([\\d\\.]+)(\\))" + SM + 
"(second|minute|hour|day|month|year)", Pattern.CASE_INSENSITIVE);
 private static final Pattern PTN_CONCAT = 
Pattern.compile("concat\\(.+?\\)");//non-greedy
 private static final Pattern PTN_HAVING_ESCAPE_FUNCTION = 
Pattern.compile("\\{fn" + "(.*?)" + "\\}", Pattern.CASE_INSENSITIVE);

[3/7] kylin git commit: refine StorageCleanupJob

[mahongbin]

refine StorageCleanupJob


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2dffc1dc
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2dffc1dc
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2dffc1dc

Branch: refs/heads/master
Commit: 2dffc1dc47f480f374a119f02f95d8e5da84d486
Parents: 99eb648
Author: Hongbin Ma 
Authored: Tue Nov 1 13:31:04 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Nov 3 13:34:12 2016 +0800

--
 .../kylin/storage/hbase/util/StorageCleanupJob.java   | 10 +-
 1 file changed, 9 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/2dffc1dc/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
index 2c2f11c..dcd90e9 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
@@ -260,16 +260,21 @@ public class StorageCleanupJob extends 
AbstractApplication {
 List allHiveTablesNeedToBeDeleted = new ArrayList();
 List workingJobList = new ArrayList();
 
+StringBuilder sb = new StringBuilder();
 for (String jobId : allJobs) {
 // only remove FINISHED and DISCARDED job intermediate table
 final ExecutableState state = 
executableManager.getOutput(jobId).getState();
 if (!state.isFinalState()) {
 workingJobList.add(jobId);
-logger.info("Skip intermediate hive table with job id " + 
jobId + " with job status " + state);
+sb.append(jobId).append("(").append(state).append("), ");
 }
 }
+logger.info("Working jobIDs: " + workingJobList);
 
 while ((line = reader.readLine()) != null) {
+
+logger.info("Checking table " + line);
+
 if (!line.startsWith(preFix))
 continue;
 
@@ -288,12 +293,15 @@ public class StorageCleanupJob extends 
AbstractApplication {
 if (UUId_PATTERN.matcher(uuid).matches()) {
 //Check whether it's a hive table in use
 if (isTableInUse(uuid, workingJobList)) {
+logger.info("Skip because not isTableInUse");
 isNeedDel = false;
 }
 } else {
+logger.info("Skip because not match pattern");
 isNeedDel = false;
 }
 } else {
+logger.info("Skip because length not qualified");
 isNeedDel = false;
 }
 

[1/7] kylin git commit: KYLIN-2154 source table rows will be skipped if TOPN's group column contains NULL values

[mahongbin]

Repository: kylin
Updated Branches:
  refs/heads/master d9c118bcc -> f9adf10d2


KYLIN-2154 source table rows will be skipped if TOPN's group column contains 
NULL values


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f9adf10d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f9adf10d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f9adf10d

Branch: refs/heads/master
Commit: f9adf10d2574bb244e1cbd18bf0f5185e044213b
Parents: e3e9ad3
Author: Hongbin Ma 
Authored: Wed Nov 2 22:28:35 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Nov 3 13:34:12 2016 +0800

--
 .../java/org/apache/kylin/measure/topn/TopNMeasureType.java | 9 +++--
 .../apache/kylin/storage/gtrecord/CubeSegmentScanner.java   | 5 -
 2 files changed, 11 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/f9adf10d/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
 
b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
index 7e788e4..f34c57c 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
@@ -18,6 +18,7 @@
 
 package org.apache.kylin.measure.topn;
 
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.Iterator;
 import java.util.List;
@@ -142,8 +143,12 @@ public class TopNMeasureType extends 
MeasureType {
 final ByteArray key = new ByteArray(keyLength);
 int offset = 0;
 for (int i = 0; i < dimensionEncodings.length; i++) {
-byte[] valueBytes = Bytes.toBytes(values[i + 1]);
-dimensionEncodings[i].encode(valueBytes, 
valueBytes.length, key.array(), offset);
+if (values[i + 1] == null) {
+Arrays.fill(key.array(), offset, offset + 
dimensionEncodings[i].getLengthOfEncoding(), DimensionEncoding.NULL);
+} else {
+byte[] valueBytes = Bytes.toBytes(values[i + 1]);
+dimensionEncodings[i].encode(valueBytes, 
valueBytes.length, key.array(), offset);
+}
 offset += dimensionEncodings[i].getLengthOfEncoding();
 }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/f9adf10d/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
--
diff --git 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
index 6e19c09..c6a6daa 100644
--- 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
+++ 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
@@ -53,6 +53,9 @@ public class CubeSegmentScanner implements IGTScanner {
 
 public CubeSegmentScanner(CubeSegment cubeSeg, Cuboid cuboid, 
Set dimensions, Set groups, //
 Collection metrics, TupleFilter originalfilter, 
StorageContext context, String gtStorage) {
+
+logger.info("Init CubeSegmentScanner for segment {}", 
cubeSeg.getName());
+
 this.cuboid = cuboid;
 this.cubeSeg = cubeSeg;
 
@@ -61,7 +64,7 @@ public class CubeSegmentScanner implements IGTScanner {
 //is working on its own copy
 byte[] serialize = TupleFilterSerializer.serialize(originalfilter, 
StringCodeSystem.INSTANCE);
 TupleFilter filter = TupleFilterSerializer.deserialize(serialize, 
StringCodeSystem.INSTANCE);
-
+
 // translate FunctionTupleFilter to IN clause
 ITupleFilterTransformer translator = new 
BuiltInFunctionTransformer(cubeSeg.getDimensionEncodingMap());
 filter = translator.transform(filter);

kylin git commit: KYLIN-2155 get-properties.sh doesn't support parameters starting with -n (by Ricardo Jorge de Sousa Teixeira)

[mahongbin]

Repository: kylin
Updated Branches:
  refs/heads/master e8d1e345b -> f93047233


KYLIN-2155 get-properties.sh doesn't support parameters starting with -n (by 
Ricardo Jorge de Sousa Teixeira)


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f9304723
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f9304723
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f9304723

Branch: refs/heads/master
Commit: f930472338a7cd6a5e07c7b6a7082e590fe06464
Parents: e8d1e34
Author: Hongbin Ma 
Authored: Thu Nov 3 10:38:02 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Nov 3 10:38:02 2016 +0800

--
 build/bin/get-properties.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/f9304723/build/bin/get-properties.sh
--
diff --git a/build/bin/get-properties.sh b/build/bin/get-properties.sh
index 516e8c9..79f5030 100755
--- a/build/bin/get-properties.sh
+++ b/build/bin/get-properties.sh
@@ -24,4 +24,4 @@ then
 fi
 
 result=`cat ${KYLIN_HOME}/conf/kylin.properties | grep -w "^$1" | grep -v '^#' 
| awk -F= '{ n = index($0,"="); print substr($0,n+1)}' | cut -c 1- |tail -1`
-echo $result
+echo "$result"

kylin git commit: minor, extract job conf in diagnosis

[lidong]

Repository: kylin
Updated Branches:
  refs/heads/v1.6.0-rc1 bc2ec1bf1 -> 669482d19


minor, extract job conf in diagnosis


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/669482d1
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/669482d1
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/669482d1

Branch: refs/heads/v1.6.0-rc1
Commit: 669482d19bddb437e72c4c396f51f42c00de03a5
Parents: bc2ec1b
Author: lidongsjtu 
Authored: Tue Nov 1 21:58:02 2016 +0800
Committer: lidongsjtu 
Committed: Wed Nov 2 22:52:19 2016 +0800

--
 .../kylin/source/hive/HiveCmdBuilder.java   |   2 +-
 .../apache/kylin/tool/JobDiagnosisInfoCLI.java  |  35 ++--
 .../kylin/tool/JobTaskCounterExtractor.java | 155 ---
 .../apache/kylin/tool/MrJobInfoExtractor.java   | 188 +++
 4 files changed, 207 insertions(+), 173 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/669482d1/source-hive/src/main/java/org/apache/kylin/source/hive/HiveCmdBuilder.java
--
diff --git 
a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveCmdBuilder.java 
b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveCmdBuilder.java
index 5a5b4e0..bce85b8 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveCmdBuilder.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveCmdBuilder.java
@@ -80,7 +80,7 @@ public class HiveCmdBuilder {
 
 logger.info("The statements to execute in beeline: \n" + 
hqlBuf);
 if (logger.isDebugEnabled()) {
-logger.debug("THe SQL to execute in beeline: \n" + 
IOUtils.toString(new FileReader(tmpHql)));
+logger.debug("The SQL to execute in beeline: \n" + 
IOUtils.toString(new FileReader(tmpHql)));
 }
 } catch (IOException e) {
 throw new RuntimeException(e);

http://git-wip-us.apache.org/repos/asf/kylin/blob/669482d1/tool/src/main/java/org/apache/kylin/tool/JobDiagnosisInfoCLI.java
--
diff --git a/tool/src/main/java/org/apache/kylin/tool/JobDiagnosisInfoCLI.java 
b/tool/src/main/java/org/apache/kylin/tool/JobDiagnosisInfoCLI.java
index c8ff1f4..638d97b 100644
--- a/tool/src/main/java/org/apache/kylin/tool/JobDiagnosisInfoCLI.java
+++ b/tool/src/main/java/org/apache/kylin/tool/JobDiagnosisInfoCLI.java
@@ -87,7 +87,7 @@ public class JobDiagnosisInfoCLI extends 
AbstractInfoExtractor {
 
 @Override
 protected void executeExtract(OptionsHelper optionsHelper, File exportDir) 
throws Exception {
-String jobId = optionsHelper.getOptionValue(OPTION_JOB_ID);
+String kylinJobId = optionsHelper.getOptionValue(OPTION_JOB_ID);
 boolean includeCube = optionsHelper.hasOption(OPTION_INCLUDE_CUBE) ? 
Boolean.valueOf(optionsHelper.getOptionValue(OPTION_INCLUDE_CUBE)) : true;
 boolean includeYarnLogs = 
optionsHelper.hasOption(OPTION_INCLUDE_YARN_LOGS) ? 
Boolean.valueOf(optionsHelper.getOptionValue(OPTION_INCLUDE_YARN_LOGS)) : true;
 boolean includeClient = optionsHelper.hasOption(OPTION_INCLUDE_CLIENT) 
? Boolean.valueOf(optionsHelper.getOptionValue(OPTION_INCLUDE_CLIENT)) : true;
@@ -95,14 +95,14 @@ public class JobDiagnosisInfoCLI extends 
AbstractInfoExtractor {
 
 // dump job output
 logger.info("Start to dump job output");
-ExecutablePO executablePO = executableDao.getJob(jobId);
-addRequired(ResourceStore.EXECUTE_RESOURCE_ROOT + "/" + jobId);
-addRequired(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT + "/" + jobId);
-for (ExecutablePO task : executablePO.getTasks()) {
-addRequired(ResourceStore.EXECUTE_RESOURCE_ROOT + "/" + 
task.getUuid());
-addRequired(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT + "/" + 
task.getUuid());
+ExecutablePO executablePO = executableDao.getJob(kylinJobId);
+addRequired(ResourceStore.EXECUTE_RESOURCE_ROOT + "/" + kylinJobId);
+addRequired(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT + "/" + 
kylinJobId);
+for (ExecutablePO kylinTask : executablePO.getTasks()) {
+addRequired(ResourceStore.EXECUTE_RESOURCE_ROOT + "/" + 
kylinTask.getUuid());
+addRequired(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT + "/" + 
kylinTask.getUuid());
 if (includeYarnLogs) {
-yarnLogsResources.add(task.getUuid());
+yarnLogsResources.add(kylinTask.getUuid());
 }
 }
 extractResources(exportDir);
@@ -121,14 +121,14 @@ public class JobDiagnosisInfoCLI extends 
AbstractInfoExtractor {
 }
 }
 
-// 

kylin git commit: KYLIN-2141: Add program-friendly interfaces for ResourceTool

[liyang]

Repository: kylin
Updated Branches:
  refs/heads/yang21 b662a9543 -> f3ac9ff4f


KYLIN-2141: Add program-friendly interfaces for ResourceTool

Signed-off-by: Yang Li 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f3ac9ff4
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f3ac9ff4
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f3ac9ff4

Branch: refs/heads/yang21
Commit: f3ac9ff4f20f5d7e4b28eaff99b388f796594824
Parents: b662a95
Author: Yiming Liu 
Authored: Mon Oct 31 17:45:09 2016 +0800
Committer: Yang Li 
Committed: Wed Nov 2 22:03:51 2016 +0800

--
 .../kylin/common/persistence/ResourceTool.java  | 28 +---
 1 file changed, 24 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/f3ac9ff4/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
 
b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
index 2f16eb7..2c8bc83 100644
--- 
a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
+++ 
b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
@@ -49,11 +49,11 @@ public class ResourceTool {
 
 String include = System.getProperty("include");
 if (include != null) {
-includes = include.split("\\s*,\\s*");
+setIncludes(include.split("\\s*,\\s*"));
 }
 String exclude = System.getProperty("exclude");
 if (exclude != null) {
-excludes = exclude.split("\\s*,\\s*");
+setExcludes(exclude.split("\\s*,\\s*"));
 }
 
 String cmd = args[0];
@@ -84,26 +84,46 @@ public class ResourceTool {
 }
 }
 
-public static void cat(KylinConfig config, String path) throws IOException 
{
+public static String[] getIncludes() {
+return includes;
+}
+
+public static void setIncludes(String[] arg) {
+includes = arg;
+}
+
+public static String[] getExcludes() {
+return excludes;
+}
+
+public static void setExcludes(String[] arg) {
+excludes = arg;
+}
+
+public static String cat(KylinConfig config, String path) throws 
IOException {
 ResourceStore store = ResourceStore.getStore(config);
 InputStream is = store.getResource(path).inputStream;
 BufferedReader br = null;
+StringBuffer sb = new StringBuffer();
 String line;
 try {
 br = new BufferedReader(new InputStreamReader(is));
 while ((line = br.readLine()) != null) {
 System.out.println(line);
+sb.append(line).append('\n');
 }
 } finally {
 IOUtils.closeQuietly(is);
 IOUtils.closeQuietly(br);
 }
+return sb.toString();
 }
 
-public static void list(KylinConfig config, String path) throws 
IOException {
+public static NavigableSet list(KylinConfig config, String path) 
throws IOException {
 ResourceStore store = ResourceStore.getStore(config);
 NavigableSet result = store.listResources(path);
 System.out.println("" + result);
+return result;
 }
 
 public static void copy(KylinConfig srcConfig, KylinConfig dstConfig, 
String path) throws IOException {

kylin git commit: KYLIN-1726 undo ‘task discard itself’ changes as it is not needed

[shaofengshi]

Repository: kylin
Updated Branches:
  refs/heads/v1.6.0-rc1 7a3793059 -> bc2ec1bf1


KYLIN-1726 undo ‘task discard itself’ changes as it is not needed

Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/bc2ec1bf
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/bc2ec1bf
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/bc2ec1bf

Branch: refs/heads/v1.6.0-rc1
Commit: bc2ec1bf17d63f7a5b9884ddc77fec6041d2b818
Parents: 7a37930
Author: shaofengshi 
Authored: Wed Nov 2 18:27:57 2016 +0800
Committer: shaofengshi 
Committed: Wed Nov 2 18:28:50 2016 +0800

--
 .../kylin/job/execution/AbstractExecutable.java |  2 -
 .../job/execution/DefaultChainedExecutable.java |  2 -
 .../kylin/job/execution/ExecuteResult.java  |  4 --
 .../kylin/job/DiscardedTestExecutable.java  | 41 
 .../impl/threadpool/DefaultSchedulerTest.java   | 16 
 .../kylin/provision/BuildCubeWithStream.java|  6 +--
 6 files changed, 1 insertion(+), 70 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/bc2ec1bf/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
--
diff --git 
a/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java 
b/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
index b4ca469..90e4d3c 100644
--- 
a/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
+++ 
b/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
@@ -74,8 +74,6 @@ public abstract class AbstractExecutable implements 
Executable, Idempotent {
 if (!isDiscarded()) {
 if (result.succeed()) {
 executableManager.updateJobOutput(getId(), 
ExecutableState.SUCCEED, null, result.output());
-} else if (result.discarded()) {
-executableManager.updateJobOutput(getId(), 
ExecutableState.DISCARDED, null, result.output());
 } else {
 executableManager.updateJobOutput(getId(), 
ExecutableState.ERROR, null, result.output());
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/bc2ec1bf/core-job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java
--
diff --git 
a/core-job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java
 
b/core-job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java
index 5a57b05..39a5f4f 100644
--- 
a/core-job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java
+++ 
b/core-job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java
@@ -119,8 +119,6 @@ public class DefaultChainedExecutable extends 
AbstractExecutable implements Chai
 } else {
 jobService.updateJobOutput(getId(), ExecutableState.READY, 
null, null);
 }
-} else if (result.discarded()) {
-jobService.updateJobOutput(getId(), ExecutableState.DISCARDED, 
null, result.output());
 } else {
 setEndTime(System.currentTimeMillis());
 jobService.updateJobOutput(getId(), ExecutableState.ERROR, null, 
result.output());

http://git-wip-us.apache.org/repos/asf/kylin/blob/bc2ec1bf/core-job/src/main/java/org/apache/kylin/job/execution/ExecuteResult.java
--
diff --git 
a/core-job/src/main/java/org/apache/kylin/job/execution/ExecuteResult.java 
b/core-job/src/main/java/org/apache/kylin/job/execution/ExecuteResult.java
index 2347e7d..760a574 100644
--- a/core-job/src/main/java/org/apache/kylin/job/execution/ExecuteResult.java
+++ b/core-job/src/main/java/org/apache/kylin/job/execution/ExecuteResult.java
@@ -49,10 +49,6 @@ public final class ExecuteResult {
 return state == State.SUCCEED;
 }
 
-public boolean discarded() {
-return state == State.DISCARDED;
-}
-
 public String output() {
 return output;
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/bc2ec1bf/core-job/src/test/java/org/apache/kylin/job/DiscardedTestExecutable.java
--
diff --git 
a/core-job/src/test/java/org/apache/kylin/job/DiscardedTestExecutable.java 
b/core-job/src/test/java/org/apache/kylin/job/DiscardedTestExecutable.java
deleted file mode 100644
index 9362e18..000
--- a/core-job/src/test/java/org/apache/kylin/job/DiscardedTestExecutable.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  

kylin git commit: KYLIN-1726 undo ‘task discard itself’ changes as it is not needed

[shaofengshi]

Repository: kylin
Updated Branches:
  refs/heads/master 5ffc5fa0a -> f8d3f1430


KYLIN-1726 undo ‘task discard itself’ changes as it is not needed

Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f8d3f143
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f8d3f143
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f8d3f143

Branch: refs/heads/master
Commit: f8d3f143079719b43a5758d24368ab518238dccc
Parents: 5ffc5fa
Author: shaofengshi 
Authored: Wed Nov 2 18:27:57 2016 +0800
Committer: shaofengshi 
Committed: Wed Nov 2 18:27:57 2016 +0800

--
 .../kylin/job/execution/AbstractExecutable.java |  2 -
 .../job/execution/DefaultChainedExecutable.java |  2 -
 .../kylin/job/execution/ExecuteResult.java  |  4 --
 .../kylin/job/DiscardedTestExecutable.java  | 41 
 .../impl/threadpool/DefaultSchedulerTest.java   | 16 
 .../kylin/provision/BuildCubeWithStream.java|  6 +--
 6 files changed, 1 insertion(+), 70 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/f8d3f143/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
--
diff --git 
a/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java 
b/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
index b4ca469..90e4d3c 100644
--- 
a/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
+++ 
b/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
@@ -74,8 +74,6 @@ public abstract class AbstractExecutable implements 
Executable, Idempotent {
 if (!isDiscarded()) {
 if (result.succeed()) {
 executableManager.updateJobOutput(getId(), 
ExecutableState.SUCCEED, null, result.output());
-} else if (result.discarded()) {
-executableManager.updateJobOutput(getId(), 
ExecutableState.DISCARDED, null, result.output());
 } else {
 executableManager.updateJobOutput(getId(), 
ExecutableState.ERROR, null, result.output());
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/f8d3f143/core-job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java
--
diff --git 
a/core-job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java
 
b/core-job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java
index 5a57b05..39a5f4f 100644
--- 
a/core-job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java
+++ 
b/core-job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java
@@ -119,8 +119,6 @@ public class DefaultChainedExecutable extends 
AbstractExecutable implements Chai
 } else {
 jobService.updateJobOutput(getId(), ExecutableState.READY, 
null, null);
 }
-} else if (result.discarded()) {
-jobService.updateJobOutput(getId(), ExecutableState.DISCARDED, 
null, result.output());
 } else {
 setEndTime(System.currentTimeMillis());
 jobService.updateJobOutput(getId(), ExecutableState.ERROR, null, 
result.output());

http://git-wip-us.apache.org/repos/asf/kylin/blob/f8d3f143/core-job/src/main/java/org/apache/kylin/job/execution/ExecuteResult.java
--
diff --git 
a/core-job/src/main/java/org/apache/kylin/job/execution/ExecuteResult.java 
b/core-job/src/main/java/org/apache/kylin/job/execution/ExecuteResult.java
index 2347e7d..760a574 100644
--- a/core-job/src/main/java/org/apache/kylin/job/execution/ExecuteResult.java
+++ b/core-job/src/main/java/org/apache/kylin/job/execution/ExecuteResult.java
@@ -49,10 +49,6 @@ public final class ExecuteResult {
 return state == State.SUCCEED;
 }
 
-public boolean discarded() {
-return state == State.DISCARDED;
-}
-
 public String output() {
 return output;
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/f8d3f143/core-job/src/test/java/org/apache/kylin/job/DiscardedTestExecutable.java
--
diff --git 
a/core-job/src/test/java/org/apache/kylin/job/DiscardedTestExecutable.java 
b/core-job/src/test/java/org/apache/kylin/job/DiscardedTestExecutable.java
deleted file mode 100644
index 9362e18..000
--- a/core-job/src/test/java/org/apache/kylin/job/DiscardedTestExecutable.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the 

[3/5] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[mahongbin]

http://git-wip-us.apache.org/repos/asf/kylin/blob/6770b421/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
index 0e95102..c59fb33 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
@@ -27,7 +27,8 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Table;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.persistence.ResourceStore;
 import org.apache.kylin.cube.CubeSegment;
@@ -56,7 +57,7 @@ public class HBaseStreamingOutput implements IStreamingOutput 
{
 try {
 CubeSegment cubeSegment = (CubeSegment) buildable;
 
-final HTableInterface hTable;
+final Table hTable;
 hTable = createHTable(cubeSegment);
 List cuboidWriters = Lists.newArrayList();
 cuboidWriters.add(new HBaseCuboidWriter(cubeSegment, hTable));
@@ -88,10 +89,10 @@ public class HBaseStreamingOutput implements 
IStreamingOutput {
 }
 }
 
-private HTableInterface createHTable(final CubeSegment cubeSegment) throws 
IOException {
+private Table createHTable(final CubeSegment cubeSegment) throws 
IOException {
 final String hTableName = cubeSegment.getStorageLocationIdentifier();
 CubeHTableUtil.createHTable(cubeSegment, null);
-final HTableInterface hTable = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(hTableName);
+final Table hTable = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(TableName.valueOf(hTableName));
 logger.info("hTable:" + hTableName + " for segment:" + 
cubeSegment.getName() + " created!");
 return hTable;
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/6770b421/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-if (admin.isTableEnabled(table)) {
-admin.disableTable(table);
+

[4/5] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[mahongbin]

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6770b421
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6770b421
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6770b421

Branch: refs/heads/yang21-cdh5.7
Commit: 6770b421a65e8d131709b27d91b09c9dfb5ebc4d
Parents: b662a95
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: Hongbin Ma 
Committed: Wed Nov 2 17:55:12 2016 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 13 ++-
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 36 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  6 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 23 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../hbase/steps/HBaseStreamingOutput.java   |  9 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 36 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 22 ++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 36 
 .../kylin/tool/CubeMigrationCheckCLI.java   | 16 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 .../apache/kylin/tool/StorageCleanupJob.java| 20 +++--
 53 files changed, 450 insertions(+), 500 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/6770b421/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-  

[2/5] kylin git commit: minor fix for QueryUtil

[mahongbin]

minor fix for QueryUtil


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b662a954
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b662a954
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b662a954

Branch: refs/heads/yang21-cdh5.7
Commit: b662a95437524449dde2113d0b84505d0eaf01d6
Parents: 37fbf0c
Author: Hongbin Ma 
Authored: Wed Nov 2 15:29:56 2016 +0800
Committer: Hongbin Ma 
Committed: Wed Nov 2 17:32:49 2016 +0800

--
 .../src/main/java/org/apache/kylin/rest/util/QueryUtil.java| 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/b662a954/server-base/src/main/java/org/apache/kylin/rest/util/QueryUtil.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/util/QueryUtil.java 
b/server-base/src/main/java/org/apache/kylin/rest/util/QueryUtil.java
index 66619fe..eb73e8f 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/util/QueryUtil.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/util/QueryUtil.java
@@ -94,7 +94,7 @@ public class QueryUtil {
 private static final String SM = "\\s+";
 private static final Pattern PTN_GROUP_BY = Pattern.compile(S1 + 
"GROUP" + SM + "BY" + S1, Pattern.CASE_INSENSITIVE);
 private static final Pattern PTN_HAVING_COUNT_GREATER_THAN_ZERO = 
Pattern.compile(S1 + "HAVING" + SM + "[(]?" + S0 + "COUNT" + S0 + "[(]" + S0 + 
"1" + S0 + "[)]" + S0 + ">" + S0 + "0" + S0 + "[)]?", Pattern.CASE_INSENSITIVE);
-private static final Pattern PTN_SUM_1 = Pattern.compile(S1 + "SUM" + 
S0 + "[(]" + S0 + "[1]" + S0 + "[)]" + S1, Pattern.CASE_INSENSITIVE);
+private static final Pattern PTN_SUM_1 = Pattern.compile(S0 + "SUM" + 
S0 + "[(]" + S0 + "[1]" + S0 + "[)]" + S0, Pattern.CASE_INSENSITIVE);
 private static final Pattern PTN_INTERVAL = Pattern.compile("interval" 
+ SM + "(floor\\()([\\d\\.]+)(\\))" + SM + 
"(second|minute|hour|day|month|year)", Pattern.CASE_INSENSITIVE);
 private static final Pattern PTN_CONCAT = 
Pattern.compile("concat\\(.+?\\)");//non-greedy
 private static final Pattern PTN_HAVING_ESCAPE_FUNCTION = 
Pattern.compile("\\{fn" + "(.*?)" + "\\}", Pattern.CASE_INSENSITIVE);

[5/5] kylin git commit: KYLIN-1672 support kylin on cdh 5.7

[mahongbin]

KYLIN-1672 support kylin on cdh 5.7

Signed-off-by: Li Yang 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/feb963b1
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/feb963b1
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/feb963b1

Branch: refs/heads/yang21-cdh5.7
Commit: feb963b136bb1356890685d443bddd1534968b3e
Parents: 6770b42
Author: Lynne Jiang 
Authored: Mon May 16 03:33:27 2016 -0700
Committer: Hongbin Ma 
Committed: Wed Nov 2 17:57:11 2016 +0800

--
 build/conf/kylin.properties |   3 +
 dev-support/test_all_against_hdp_2_2_4_2_2.sh   |   0
 .../kylin/engine/mr/steps/MockupMapContext.java |  15 +-
 examples/test_case_data/sandbox/core-site.xml   | 146 +++---
 examples/test_case_data/sandbox/hbase-site.xml  | 162 ++
 examples/test_case_data/sandbox/hdfs-site.xml   | 259 ++
 examples/test_case_data/sandbox/mapred-site.xml | 398 ++-
 examples/test_case_data/sandbox/yarn-site.xml   | 496 ++-
 pom.xml |  16 +-
 server/pom.xml  |  36 ++
 .../storage/hbase/steps/MockupMapContext.java   |  19 +-
 tool/pom.xml|  12 +
 12 files changed, 431 insertions(+), 1131 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/feb963b1/build/conf/kylin.properties
--
diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties
index ed86bdb..ebd8cde 100644
--- a/build/conf/kylin.properties
+++ b/build/conf/kylin.properties
@@ -118,6 +118,9 @@ kylin.job.mapreduce.mapper.input.rows=100
 
 kylin.job.step.timeout=7200
 
+# for secure cdh, filtering hive dependency is risky, so filter nothing
+kylin.job.dependency.filterlist=[^,]+
+
 ### CUBE ###
 
 # 'auto', 'inmem', 'layer' or 'random' for testing

http://git-wip-us.apache.org/repos/asf/kylin/blob/feb963b1/dev-support/test_all_against_hdp_2_2_4_2_2.sh
--
diff --git a/dev-support/test_all_against_hdp_2_2_4_2_2.sh 
b/dev-support/test_all_against_hdp_2_2_4_2_2.sh
old mode 100644
new mode 100755

http://git-wip-us.apache.org/repos/asf/kylin/blob/feb963b1/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
--
diff --git 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
index 847071d..9900465 100644
--- 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
+++ 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
@@ -77,6 +77,7 @@ public class MockupMapContext {
 outKV[0] = key;
 outKV[1] = value;
 }
+
 }
 
 @Override
@@ -99,6 +100,7 @@ public class MockupMapContext {
 throw new NotImplementedException();
 }
 
+
 @Override
 public float getProgress() {
 throw new NotImplementedException();
@@ -195,17 +197,17 @@ public class MockupMapContext {
 }
 
 @Override
-public RawComparator getSortComparator() {
+public boolean userClassesTakesPrecedence() {
 throw new NotImplementedException();
 }
 
 @Override
-public String getJar() {
+public RawComparator getSortComparator() {
 throw new NotImplementedException();
 }
 
 @Override
-public RawComparator getGroupingComparator() {
+public String getJar() {
 throw new NotImplementedException();
 }
 
@@ -221,7 +223,7 @@ public class MockupMapContext {
 
 @Override
 public boolean getProfileEnabled() {
-throw new NotImplementedException();
+return false;
 }
 
 @Override
@@ -308,6 +310,11 @@ public class MockupMapContext {
 public RawComparator getCombinerKeyGroupingComparator() {
 throw new NotImplementedException();
 }
+
+@Override
+public RawComparator getGroupingComparator() {
+return null;
+}
 });
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/feb963b1/examples/test_case_data/sandbox/core-site.xml
--
diff --git a/examples/test_case_data/sandbox/core-site.xml 

[4/4] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[mahongbin]

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6770b421
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6770b421
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6770b421

Branch: refs/heads/yang21-hbase1.x
Commit: 6770b421a65e8d131709b27d91b09c9dfb5ebc4d
Parents: b662a95
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: Hongbin Ma 
Committed: Wed Nov 2 17:55:12 2016 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 13 ++-
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 36 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  6 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 23 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../hbase/steps/HBaseStreamingOutput.java   |  9 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 36 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 22 ++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 36 
 .../kylin/tool/CubeMigrationCheckCLI.java   | 16 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 .../apache/kylin/tool/StorageCleanupJob.java| 20 +++--
 53 files changed, 450 insertions(+), 500 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/6770b421/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-

[2/4] kylin git commit: minor fix for QueryUtil

[mahongbin]

minor fix for QueryUtil


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b662a954
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b662a954
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b662a954

Branch: refs/heads/yang21-hbase1.x
Commit: b662a95437524449dde2113d0b84505d0eaf01d6
Parents: 37fbf0c
Author: Hongbin Ma 
Authored: Wed Nov 2 15:29:56 2016 +0800
Committer: Hongbin Ma 
Committed: Wed Nov 2 17:32:49 2016 +0800

--
 .../src/main/java/org/apache/kylin/rest/util/QueryUtil.java| 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/b662a954/server-base/src/main/java/org/apache/kylin/rest/util/QueryUtil.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/util/QueryUtil.java 
b/server-base/src/main/java/org/apache/kylin/rest/util/QueryUtil.java
index 66619fe..eb73e8f 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/util/QueryUtil.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/util/QueryUtil.java
@@ -94,7 +94,7 @@ public class QueryUtil {
 private static final String SM = "\\s+";
 private static final Pattern PTN_GROUP_BY = Pattern.compile(S1 + 
"GROUP" + SM + "BY" + S1, Pattern.CASE_INSENSITIVE);
 private static final Pattern PTN_HAVING_COUNT_GREATER_THAN_ZERO = 
Pattern.compile(S1 + "HAVING" + SM + "[(]?" + S0 + "COUNT" + S0 + "[(]" + S0 + 
"1" + S0 + "[)]" + S0 + ">" + S0 + "0" + S0 + "[)]?", Pattern.CASE_INSENSITIVE);
-private static final Pattern PTN_SUM_1 = Pattern.compile(S1 + "SUM" + 
S0 + "[(]" + S0 + "[1]" + S0 + "[)]" + S1, Pattern.CASE_INSENSITIVE);
+private static final Pattern PTN_SUM_1 = Pattern.compile(S0 + "SUM" + 
S0 + "[(]" + S0 + "[1]" + S0 + "[)]" + S0, Pattern.CASE_INSENSITIVE);
 private static final Pattern PTN_INTERVAL = Pattern.compile("interval" 
+ SM + "(floor\\()([\\d\\.]+)(\\))" + SM + 
"(second|minute|hour|day|month|year)", Pattern.CASE_INSENSITIVE);
 private static final Pattern PTN_CONCAT = 
Pattern.compile("concat\\(.+?\\)");//non-greedy
 private static final Pattern PTN_HAVING_ESCAPE_FUNCTION = 
Pattern.compile("\\{fn" + "(.*?)" + "\\}", Pattern.CASE_INSENSITIVE);

kylin git commit: minor fix for QueryUtil

[mahongbin]

Repository: kylin
Updated Branches:
  refs/heads/yang21 37fbf0c1e -> b662a9543


minor fix for QueryUtil


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b662a954
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b662a954
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b662a954

Branch: refs/heads/yang21
Commit: b662a95437524449dde2113d0b84505d0eaf01d6
Parents: 37fbf0c
Author: Hongbin Ma 
Authored: Wed Nov 2 15:29:56 2016 +0800
Committer: Hongbin Ma 
Committed: Wed Nov 2 17:32:49 2016 +0800

--
 .../src/main/java/org/apache/kylin/rest/util/QueryUtil.java| 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/b662a954/server-base/src/main/java/org/apache/kylin/rest/util/QueryUtil.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/util/QueryUtil.java 
b/server-base/src/main/java/org/apache/kylin/rest/util/QueryUtil.java
index 66619fe..eb73e8f 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/util/QueryUtil.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/util/QueryUtil.java
@@ -94,7 +94,7 @@ public class QueryUtil {
 private static final String SM = "\\s+";
 private static final Pattern PTN_GROUP_BY = Pattern.compile(S1 + 
"GROUP" + SM + "BY" + S1, Pattern.CASE_INSENSITIVE);
 private static final Pattern PTN_HAVING_COUNT_GREATER_THAN_ZERO = 
Pattern.compile(S1 + "HAVING" + SM + "[(]?" + S0 + "COUNT" + S0 + "[(]" + S0 + 
"1" + S0 + "[)]" + S0 + ">" + S0 + "0" + S0 + "[)]?", Pattern.CASE_INSENSITIVE);
-private static final Pattern PTN_SUM_1 = Pattern.compile(S1 + "SUM" + 
S0 + "[(]" + S0 + "[1]" + S0 + "[)]" + S1, Pattern.CASE_INSENSITIVE);
+private static final Pattern PTN_SUM_1 = Pattern.compile(S0 + "SUM" + 
S0 + "[(]" + S0 + "[1]" + S0 + "[)]" + S0, Pattern.CASE_INSENSITIVE);
 private static final Pattern PTN_INTERVAL = Pattern.compile("interval" 
+ SM + "(floor\\()([\\d\\.]+)(\\))" + SM + 
"(second|minute|hour|day|month|year)", Pattern.CASE_INSENSITIVE);
 private static final Pattern PTN_CONCAT = 
Pattern.compile("concat\\(.+?\\)");//non-greedy
 private static final Pattern PTN_HAVING_ESCAPE_FUNCTION = 
Pattern.compile("\\{fn" + "(.*?)" + "\\}", Pattern.CASE_INSENSITIVE);

[16/33] kylin git commit: KYLIN 1820 Column autocomplete should remove the user input in model designer

[shaofengshi]

KYLIN 1820 Column autocomplete should remove the user input in model designer

Signed-off-by: Jason 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5218c9f6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5218c9f6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5218c9f6

Branch: refs/heads/v1.6.0-rc1-cdh5.7
Commit: 5218c9f6bc6e27e86f29d3b75e5c6aaa73a9cb98
Parents: 647d51e
Author: chenzhx <346839...@qq.com>
Authored: Wed Oct 26 11:35:15 2016 +0800
Committer: Jason 
Committed: Mon Oct 31 16:59:15 2016 +0800

--
 webapp/app/js/directives/select.js | 939 
 1 file changed, 703 insertions(+), 236 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/5218c9f6/webapp/app/js/directives/select.js
--
diff --git a/webapp/app/js/directives/select.js 
b/webapp/app/js/directives/select.js
index c8cf6a2..7327af9 100644
--- a/webapp/app/js/directives/select.js
+++ b/webapp/app/js/directives/select.js
@@ -1,14 +1,13 @@
 /*!
  * ui-select
  * http://github.com/angular-ui/ui-select
- * Version: 0.13.2 - 2015-10-09T15:34:24.040Z
+ * Version: 0.19.5 - 2016-10-24T23:13:59.434Z
  * License: MIT
  */
 
 
 (function () {
 "use strict";
-
 var KEY = {
 TAB: 9,
 ENTER: 13,
@@ -42,7 +41,7 @@ var KEY = {
 return true;
 }
 
-if (e.metaKey) return true;
+if (e.metaKey || e.ctrlKey || e.altKey) return true;
 
 return false;
 },
@@ -55,6 +54,13 @@ var KEY = {
 },
 isHorizontalMovement: function (k){
   return ~[KEY.LEFT,KEY.RIGHT,KEY.BACKSPACE,KEY.DELETE].indexOf(k);
+},
+toSeparator: function (k) {
+  var sep = {ENTER:"\n",TAB:"\t",SPACE:" "}[k];
+  if (sep) return sep;
+  // return undefined for special keys other than enter, tab or space.
+  // no way to use them to cut strings.
+  return KEY[k] ? undefined : k;
 }
   };
 
@@ -103,11 +109,16 @@ var uis = angular.module('ui.select', [])
   placeholder: '', // Empty by default, like HTML tag 
   refreshDelay: 1000, // In milliseconds
   closeOnSelect: true,
+  skipFocusser: false,
   dropdownPosition: 'auto',
+  removeSelected: true,
+  resetSearchInput: true,
   generateId: function() {
 return latestId++;
   },
-  appendToBody: false
+  appendToBody: false,
+  spinnerEnabled: false,
+  spinnerClass: 'glyphicon-refresh ui-select-spin'
 })
 
 // See Rename minErr and make it accessible from outside 
https://github.com/angular/angular.js/issues/6913
@@ -139,11 +150,11 @@ var uis = angular.module('ui.select', [])
  */
 .filter('highlight', function() {
   function escapeRegexp(queryToEscape) {
-return queryToEscape.replace(/([.?*+^$[\]\\(){}|-])/g, '\\$1');
+return ('' + queryToEscape).replace(/([.?*+^$[\]\\(){}|-])/g, '\\$1');
   }
 
   return function(matchItem, query) {
-return query && matchItem ? matchItem.replace(new 
RegExp(escapeRegexp(query), 'gi'), '$&') : matchItem;
+return query && matchItem ? ('' + matchItem).replace(new 
RegExp(escapeRegexp(query), 'gi'), '$&') : matchItem;
   };
 })
 
@@ -169,8 +180,8 @@ var uis = angular.module('ui.select', [])
 }]);
 
 uis.directive('uiSelectChoices',
-  ['uiSelectConfig', 'uisRepeatParser', 'uiSelectMinErr', '$compile',
-  function(uiSelectConfig, RepeatParser, uiSelectMinErr, $compile) {
+  ['uiSelectConfig', 'uisRepeatParser', 'uiSelectMinErr', '$compile', 
'$window',
+  function(uiSelectConfig, RepeatParser, uiSelectMinErr, $compile, $window) {
 
   return {
 restrict: 'EA',
@@ -178,6 +189,9 @@ uis.directive('uiSelectChoices',
 replace: true,
 transclude: true,
 templateUrl: function(tElement) {
+  // Needed so the uiSelect can detect the transcluded content
+  tElement.addClass('ui-select-choices');
+
   // Gets theme attribute from parent (ui-select)
   var theme = tElement.parent().attr('theme') || uiSelectConfig.theme;
   return theme + '/choices.tpl.html';
@@ -187,44 +201,59 @@ uis.directive('uiSelectChoices',
 
   if (!tAttrs.repeat) throw uiSelectMinErr('repeat', "Expected 'repeat' 
expression.");
 
-  return function link(scope, element, attrs, $select, transcludeFn) {
+  // var repeat = RepeatParser.parse(attrs.repeat);
+  var groupByExp = tAttrs.groupBy;
+  var groupFilterExp = tAttrs.groupFilter;
 
-// var repeat = RepeatParser.parse(attrs.repeat);
-var groupByExp = attrs.groupBy;
-var groupFilterExp = attrs.groupFilter;
+  if (groupByExp) {
+var groups = tElement.querySelectorAll('.ui-select-choices-group');
+if (groups.length !== 1) throw uiSelectMinErr('rows', "Expected 1 
.ui-select-choices-group but got '{0}'.", groups.length);
+

[13/33] kylin git commit: remove fitler hive dependency logic in AbstractHadoopJob

[shaofengshi]

remove fitler hive dependency logic in AbstractHadoopJob


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/eabbd4e1
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/eabbd4e1
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/eabbd4e1

Branch: refs/heads/v1.6.0-rc1-cdh5.7
Commit: eabbd4e16869150625af43b3cefa5e9076378f48
Parents: 6bdd4f3
Author: Hongbin Ma 
Authored: Thu Oct 13 21:35:49 2016 +0800
Committer: Hongbin Ma 
Committed: Mon Oct 31 12:46:08 2016 +0800

--
 .../java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/eabbd4e1/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
index 417d63d..72f4437 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
@@ -200,7 +200,8 @@ public abstract class AbstractHadoopJob extends Configured 
implements Tool {
 kylinHiveDependency = kylinHiveDependency.replace(":", ",");
 
 logger.info("Hive Dependencies Before Filtered: " + 
kylinHiveDependency);
-String filteredHive = 
filterKylinHiveDependency(kylinHiveDependency);
+//String filteredHive = 
filterKylinHiveDependency(kylinHiveDependency);
+String filteredHive = kylinHiveDependency;
 logger.info("Hive Dependencies After Filtered: " + filteredHive);
 
 StringUtil.appendWithSeparator(kylinDependency, filteredHive);

[08/33] kylin git commit: KYLIN-2105 remove QueryIdGenerator and QueryIdGeneratorTest, simply use UUID as QueryId

[shaofengshi]

KYLIN-2105 remove QueryIdGenerator and QueryIdGeneratorTest, simply use UUID as 
QueryId


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/46a894db
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/46a894db
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/46a894db

Branch: refs/heads/v1.6.0-rc1-cdh5.7
Commit: 46a894db60e7538468d6e7459ea760bca286249e
Parents: 5d4d639
Author: gaodayue 
Authored: Thu Oct 27 12:14:49 2016 +0800
Committer: gaodayue 
Committed: Fri Oct 28 13:32:38 2016 +0800

--
 .../apache/kylin/rest/service/QueryService.java |   8 +-
 .../kylin/rest/util/QueryIdGenerator.java   |  47 
 .../kylin/rest/util/QueryIdGeneratorTest.java   | 108 ---
 .../coprocessor/endpoint/CubeVisitService.java  |   2 +-
 4 files changed, 4 insertions(+), 161 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/46a894db/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java 
b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
index 94f2dd5..81af044 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -39,6 +39,7 @@ import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.UUID;
 
 import javax.annotation.PostConstruct;
 import javax.sql.DataSource;
@@ -72,7 +73,6 @@ import org.apache.kylin.rest.model.TableMeta;
 import org.apache.kylin.rest.request.PrepareSqlRequest;
 import org.apache.kylin.rest.request.SQLRequest;
 import org.apache.kylin.rest.response.SQLResponse;
-import org.apache.kylin.rest.util.QueryIdGenerator;
 import org.apache.kylin.rest.util.QueryUtil;
 import org.apache.kylin.rest.util.Serializer;
 import org.apache.kylin.rest.util.TableauInterceptor;
@@ -117,8 +117,6 @@ public class QueryService extends BasicService {
 private final String hbaseUrl;
 private final String userTableName;
 
-private QueryIdGenerator queryIdGenerator = new QueryIdGenerator();
-
 @Autowired
 private CacheManager cacheManager;
 
@@ -325,7 +323,7 @@ public class QueryService extends BasicService {
 throw new InternalErrorException("Query is not allowed in " + 
serverMode + " mode.");
 }
 
-final String queryId = 
queryIdGenerator.nextId(sqlRequest.getProject());
+final String queryId = UUID.randomUUID().toString();
 
 Map toggles = new HashMap<>();
 toggles.put(BackdoorToggles.KEY_QUERY_ID, queryId);
@@ -334,7 +332,7 @@ public class QueryService extends BasicService {
 }
 BackdoorToggles.setToggles(toggles);
 
-try (SetThreadName ignored = new SetThreadName("Query-%s", queryId)) {
+try (SetThreadName ignored = new SetThreadName("Query %s", queryId)) {
 String sql = sqlRequest.getSql();
 String project = sqlRequest.getProject();
 logger.info("Using project: " + project);

http://git-wip-us.apache.org/repos/asf/kylin/blob/46a894db/server-base/src/main/java/org/apache/kylin/rest/util/QueryIdGenerator.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/util/QueryIdGenerator.java 
b/server-base/src/main/java/org/apache/kylin/rest/util/QueryIdGenerator.java
deleted file mode 100644
index 2dd19c2..000
--- a/server-base/src/main/java/org/apache/kylin/rest/util/QueryIdGenerator.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.rest.util;
-
-import org.apache.commons.lang3.time.FastDateFormat;
-
-import javax.annotation.Nonnull;
-import javax.annotation.concurrent.ThreadSafe;
-import 

[27/33] kylin git commit: KYLIN-2122 Move the partition offset calculation before submitting job

[shaofengshi]

KYLIN-2122 Move the partition offset calculation before submitting job


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2beccbf9
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2beccbf9
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2beccbf9

Branch: refs/heads/v1.6.0-rc1-cdh5.7
Commit: 2beccbf94fd8e43c072959aeec98cc05d8552788
Parents: b6f608f
Author: shaofengshi 
Authored: Tue Oct 25 11:54:22 2016 +0800
Committer: shaofengshi 
Committed: Wed Nov 2 13:47:34 2016 +0800

--
 .../org/apache/kylin/cube/CubeInstance.java |   9 ++
 .../java/org/apache/kylin/cube/CubeManager.java | 103 +-
 .../org/apache/kylin/cube/CubeManagerTest.java  |   2 +-
 .../org/apache/kylin/cube/CubeSegmentsTest.java |   2 +-
 .../kylin/job/constant/ExecutableConstants.java |   1 +
 .../java/org/apache/kylin/source/ISource.java   |  13 ++-
 .../org/apache/kylin/source/SourceFactory.java  |   2 +-
 .../apache/kylin/source/SourcePartition.java| 103 ++
 .../kylin/engine/mr/JobBuilderSupport.java  |   2 +
 .../mr/steps/UpdateCubeInfoAfterBuildStep.java  |  67 
 .../kylin/provision/BuildCubeWithEngine.java|   8 +-
 .../kylin/provision/BuildCubeWithStream.java|  17 ++-
 .../apache/kylin/rest/service/JobService.java   |  16 +--
 .../apache/kylin/source/hive/HiveSource.java|  32 +-
 .../apache/kylin/source/kafka/KafkaMRInput.java |  90 ++--
 .../apache/kylin/source/kafka/KafkaSource.java  | 105 +-
 .../kylin/source/kafka/config/KafkaConfig.java  |   3 +
 .../source/kafka/hadoop/KafkaInputFormat.java   |  41 ---
 .../kylin/source/kafka/job/SeekOffsetStep.java  | 106 +--
 .../source/kafka/job/UpdateTimeRangeStep.java   |  78 +-
 .../kylin/source/kafka/util/KafkaClient.java|  24 -
 21 files changed, 483 insertions(+), 341 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/2beccbf9/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
index 720690d..7222457 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
@@ -439,6 +439,15 @@ public class CubeInstance extends RootPersistentEntity 
implements IRealization,
 return this.getDescriptor().getAutoMergeTimeRanges() != null && 
this.getDescriptor().getAutoMergeTimeRanges().length > 0;
 }
 
+public CubeSegment getLastSegment() {
+List existing = getSegments();
+if (existing.isEmpty()) {
+return null;
+} else {
+return existing.get(existing.size() - 1);
+}
+}
+
 @Override
 public int getSourceType() {
 return getFactTableDesc().getSourceType();

http://git-wip-us.apache.org/repos/asf/kylin/blob/2beccbf9/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index a53849e..16b468f 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -34,8 +34,6 @@ import java.util.Random;
 import java.util.UUID;
 import java.util.concurrent.ConcurrentHashMap;
 
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Maps;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.KylinConfigExt;
@@ -68,9 +66,11 @@ import 
org.apache.kylin.metadata.realization.RealizationStatusEnum;
 import org.apache.kylin.metadata.realization.RealizationType;
 import org.apache.kylin.source.ReadableTable;
 import org.apache.kylin.source.SourceFactory;
+import org.apache.kylin.source.SourcePartition;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.google.common.base.Preconditions;
 import com.google.common.collect.HashMultimap;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Multimap;
@@ -434,52 +434,20 @@ public class CubeManager implements IRealizationProvider {
 
 // append a full build segment
 public CubeSegment appendSegment(CubeInstance cube) throws IOException {
-return appendSegment(cube, 0, 0, 0, 0, null, null);
+return appendSegment(cube, 0, Long.MAX_VALUE, 0, 0, null, null);
 }
 
 public CubeSegment 

[20/33] kylin git commit: KYLIN-1726 apply kylin_job_conf.xml on KafkaFlatTableJob

[shaofengshi]

KYLIN-1726 apply kylin_job_conf.xml on KafkaFlatTableJob


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8172d0ba
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8172d0ba
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8172d0ba

Branch: refs/heads/v1.6.0-rc1-cdh5.7
Commit: 8172d0ba3e6768c803528b457db6cc3a464c8e14
Parents: a8922e8
Author: shaofengshi 
Authored: Tue Nov 1 15:27:52 2016 +0800
Committer: shaofengshi 
Committed: Tue Nov 1 15:27:52 2016 +0800

--
 .../org/apache/kylin/source/kafka/hadoop/KafkaFlatTableJob.java   | 3 +++
 1 file changed, 3 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/8172d0ba/source-kafka/src/main/java/org/apache/kylin/source/kafka/hadoop/KafkaFlatTableJob.java
--
diff --git 
a/source-kafka/src/main/java/org/apache/kylin/source/kafka/hadoop/KafkaFlatTableJob.java
 
b/source-kafka/src/main/java/org/apache/kylin/source/kafka/hadoop/KafkaFlatTableJob.java
index e20b20a..5fe6e00 100644
--- 
a/source-kafka/src/main/java/org/apache/kylin/source/kafka/hadoop/KafkaFlatTableJob.java
+++ 
b/source-kafka/src/main/java/org/apache/kylin/source/kafka/hadoop/KafkaFlatTableJob.java
@@ -18,6 +18,7 @@
 
 package org.apache.kylin.source.kafka.hadoop;
 
+import org.apache.kylin.job.engine.JobEngineConfig;
 import org.apache.kylin.source.kafka.util.KafkaClient;
 import org.apache.commons.cli.Options;
 import org.apache.hadoop.fs.Path;
@@ -97,6 +98,8 @@ public class KafkaFlatTableJob extends AbstractHadoopJob {
 throw new IllegalArgumentException("Invalid Kafka information, 
brokers " + brokers + ", topic " + topic);
 }
 
+JobEngineConfig jobEngineConfig = new 
JobEngineConfig(KylinConfig.getInstanceFromEnv());
+job.getConfiguration().addResource(new 
Path(jobEngineConfig.getHadoopJobConfFilePath(null)));
 job.getConfiguration().set(CONFIG_KAFKA_BROKERS, brokers);
 job.getConfiguration().set(CONFIG_KAFKA_TOPIC, topic);
 job.getConfiguration().set(CONFIG_KAFKA_TIMEOUT, 
String.valueOf(kafkaConfig.getTimeout()));

[07/33] kylin git commit: minor, fix concurrent issue with CubeSegment.cuboidBaseShards

[shaofengshi]

minor, fix concurrent issue with CubeSegment.cuboidBaseShards


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5d4d639d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5d4d639d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5d4d639d

Branch: refs/heads/v1.6.0-rc1-cdh5.7
Commit: 5d4d639ddb6b190e4978262e4e39778866b7270a
Parents: 8199a90
Author: gaodayue 
Authored: Wed Oct 26 21:39:11 2016 +0800
Committer: gaodayue 
Committed: Fri Oct 28 13:31:48 2016 +0800

--
 .../src/main/java/org/apache/kylin/cube/CubeSegment.java  | 7 +--
 1 file changed, 5 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/5d4d639d/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
index fdf1fb0..b3d6d6b 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
@@ -49,6 +49,8 @@ import 
com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
 
+import javax.annotation.concurrent.GuardedBy;
+
 @JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = 
Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = 
Visibility.NONE)
 public class CubeSegment implements Comparable, IBuildable, 
ISegment {
 
@@ -115,7 +117,8 @@ public class CubeSegment implements 
Comparable, IBuildable, ISegmen
 @JsonInclude(JsonInclude.Include.NON_EMPTY)
 private Map additionalInfo = new LinkedHashMap();
 
-private volatile Map cuboidBaseShards = 
Maps.newHashMap();//cuboid id ==> base(starting) shard for this cuboid
+@GuardedBy("this")
+private Map cuboidBaseShards = Maps.newHashMap(); // cuboid 
id ==> base(starting) shard for this cuboid
 
 public CubeDesc getCubeDesc() {
 return getCubeInstance().getDescriptor();
@@ -516,7 +519,7 @@ public class CubeSegment implements 
Comparable, IBuildable, ISegmen
 this.totalShards = totalShards;
 }
 
-public short getCuboidBaseShard(Long cuboidId) {
+public synchronized short getCuboidBaseShard(Long cuboidId) {
 if (totalShards > 0) {
 //shard squashed case
 

[33/33] kylin git commit: KYLIN-1672 support kylin on cdh 5.7

[shaofengshi]

KYLIN-1672 support kylin on cdh 5.7

Signed-off-by: Li Yang 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/56215d0f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/56215d0f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/56215d0f

Branch: refs/heads/v1.6.0-rc1-cdh5.7
Commit: 56215d0f0247cebf7f5efdff8516e9919963c703
Parents: 3940e80
Author: Lynne Jiang 
Authored: Mon May 16 03:33:27 2016 -0700
Committer: shaofengshi 
Committed: Wed Nov 2 16:41:16 2016 +0800

--
 dev-support/test_all_against_hdp_2_2_4_2_2.sh   |   0
 .../kylin/engine/mr/steps/MockupMapContext.java |  15 +-
 examples/test_case_data/sandbox/core-site.xml   | 146 +++---
 examples/test_case_data/sandbox/hbase-site.xml  | 162 ++
 examples/test_case_data/sandbox/hdfs-site.xml   | 259 ++
 examples/test_case_data/sandbox/mapred-site.xml | 398 ++-
 examples/test_case_data/sandbox/yarn-site.xml   | 496 ++-
 pom.xml |  16 +-
 server/pom.xml  |  36 ++
 .../storage/hbase/steps/MockupMapContext.java   |  19 +-
 tool/pom.xml|  12 +
 11 files changed, 428 insertions(+), 1131 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/56215d0f/dev-support/test_all_against_hdp_2_2_4_2_2.sh
--
diff --git a/dev-support/test_all_against_hdp_2_2_4_2_2.sh 
b/dev-support/test_all_against_hdp_2_2_4_2_2.sh
old mode 100644
new mode 100755

http://git-wip-us.apache.org/repos/asf/kylin/blob/56215d0f/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
--
diff --git 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
index 847071d..9900465 100644
--- 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
+++ 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
@@ -77,6 +77,7 @@ public class MockupMapContext {
 outKV[0] = key;
 outKV[1] = value;
 }
+
 }
 
 @Override
@@ -99,6 +100,7 @@ public class MockupMapContext {
 throw new NotImplementedException();
 }
 
+
 @Override
 public float getProgress() {
 throw new NotImplementedException();
@@ -195,17 +197,17 @@ public class MockupMapContext {
 }
 
 @Override
-public RawComparator getSortComparator() {
+public boolean userClassesTakesPrecedence() {
 throw new NotImplementedException();
 }
 
 @Override
-public String getJar() {
+public RawComparator getSortComparator() {
 throw new NotImplementedException();
 }
 
 @Override
-public RawComparator getGroupingComparator() {
+public String getJar() {
 throw new NotImplementedException();
 }
 
@@ -221,7 +223,7 @@ public class MockupMapContext {
 
 @Override
 public boolean getProfileEnabled() {
-throw new NotImplementedException();
+return false;
 }
 
 @Override
@@ -308,6 +310,11 @@ public class MockupMapContext {
 public RawComparator getCombinerKeyGroupingComparator() {
 throw new NotImplementedException();
 }
+
+@Override
+public RawComparator getGroupingComparator() {
+return null;
+}
 });
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/56215d0f/examples/test_case_data/sandbox/core-site.xml
--
diff --git a/examples/test_case_data/sandbox/core-site.xml 
b/examples/test_case_data/sandbox/core-site.xml
index 9aa588c..6162406 100644
--- a/examples/test_case_data/sandbox/core-site.xml
+++ b/examples/test_case_data/sandbox/core-site.xml
@@ -14,152 +14,146 @@
   See the License for the specific language governing permissions and
   limitations under the License.
 -->
+
 
-
 
 fs.defaultFS
-hdfs://sandbox.hortonworks.com:8020
-true
+hdfs://quickstart.cloudera:8020
 
-
 
 fs.trash.interval
-360
+1
 
-
 
-
ha.failover-controller.active-standby-elector.zk.op.retries
-120
+io.compression.codecs
+

[22/33] kylin git commit: KYLIN-2085 add more UT

[shaofengshi]

KYLIN-2085 add more UT


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3f1a495e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3f1a495e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3f1a495e

Branch: refs/heads/v1.6.0-rc1-cdh5.7
Commit: 3f1a495e1fee31a4718942d2241929ef50d4d020
Parents: 67428a5
Author: gaodayue 
Authored: Tue Nov 1 17:23:16 2016 +0800
Committer: gaodayue 
Committed: Tue Nov 1 17:26:20 2016 +0800

--
 .../common/coprocessor/FilterBaseTest.java  |   4 +-
 .../common/coprocessor/FilterSerializeTest.java | 114 ---
 2 files changed, 26 insertions(+), 92 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/3f1a495e/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterBaseTest.java
--
diff --git 
a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterBaseTest.java
 
b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterBaseTest.java
index d016e4f..291072f 100644
--- 
a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterBaseTest.java
+++ 
b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterBaseTest.java
@@ -173,8 +173,8 @@ public class FilterBaseTest extends 
LocalFileMetadataTestCase {
 return compareFilter;
 }
 
-protected CompareTupleFilter buildCompareDynamicFilter(List 
groups) {
-CompareTupleFilter compareFilter = new 
CompareTupleFilter(FilterOperatorEnum.EQ);
+protected CompareTupleFilter buildCompareDynamicFilter(List 
groups, FilterOperatorEnum operator) {
+CompareTupleFilter compareFilter = new CompareTupleFilter(operator);
 compareFilter.addChild(new ColumnTupleFilter(groups.get(0)));
 compareFilter.addChild(new DynamicTupleFilter("?0"));
 compareFilter.bindVariable("?0", "abc");

http://git-wip-us.apache.org/repos/asf/kylin/blob/3f1a495e/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterSerializeTest.java
--
diff --git 
a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterSerializeTest.java
 
b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterSerializeTest.java
index 844dbeb..e494e2e 100644
--- 
a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterSerializeTest.java
+++ 
b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterSerializeTest.java
@@ -22,7 +22,6 @@ import java.text.ParseException;
 import java.util.ArrayList;
 import java.util.List;
 
-import org.apache.kylin.metadata.filter.CompareTupleFilter;
 import org.apache.kylin.metadata.filter.LogicalTupleFilter;
 import org.apache.kylin.metadata.filter.TupleFilter;
 import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
@@ -38,11 +37,7 @@ import org.junit.Test;
  */
 public class FilterSerializeTest extends FilterBaseTest {
 
-@Test
-public void testSerialize01() {
-List groups = buildGroups();
-TupleFilter filter = buildEQCompareFilter(groups, 0);
-
+private void assertFilterSerDe(TupleFilter filter) {
 byte[] bytes = TupleFilterSerializer.serialize(filter, CS);
 TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS);
 
@@ -50,51 +45,33 @@ public class FilterSerializeTest extends FilterBaseTest {
 }
 
 @Test
-public void testSerialize02() {
-List groups = buildGroups();
-TupleFilter filter = buildEQCompareFilter(groups, 1);
-
-byte[] bytes = TupleFilterSerializer.serialize(filter, CS);
-TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS);
+public void testSerialize01() {
+assertFilterSerDe(buildEQCompareFilter(buildGroups(), 0));
+}
 
-compareFilter(filter, newFilter);
+@Test
+public void testSerialize02() {
+assertFilterSerDe(buildEQCompareFilter(buildGroups(), 1));
 }
 
 @Test
 public void testSerialize03() {
-List groups = buildGroups();
-TupleFilter filter = buildAndFilter(groups);
-
-byte[] bytes = TupleFilterSerializer.serialize(filter, CS);
-TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS);
-
-compareFilter(filter, newFilter);
+assertFilterSerDe(buildAndFilter(buildGroups()));
 }
 
 @Test
 public void testSerialize04() {
-List groups = buildGroups();
-TupleFilter filter = buildOrFilter(groups);
-
-byte[] bytes = 

[18/33] kylin git commit: KYLIN-1321 Add derived checkbox for lookup table columns on Auto Generate Dimensions panel, style update

[shaofengshi]

KYLIN-1321 Add derived checkbox for lookup table columns on Auto Generate 
Dimensions panel, style update


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/21ecfce6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/21ecfce6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/21ecfce6

Branch: refs/heads/v1.6.0-rc1-cdh5.7
Commit: 21ecfce6d0593c891c4cf7a6360c5ee29b01710c
Parents: d01a1f7
Author: Jason 
Authored: Mon Oct 31 17:08:14 2016 +0800
Committer: Jason 
Committed: Mon Oct 31 17:10:07 2016 +0800

--
 webapp/app/partials/cubeDesigner/dimensions.html | 10 +-
 1 file changed, 5 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/21ecfce6/webapp/app/partials/cubeDesigner/dimensions.html
--
diff --git a/webapp/app/partials/cubeDesigner/dimensions.html 
b/webapp/app/partials/cubeDesigner/dimensions.html
index 62ca5c9..9539250 100644
--- a/webapp/app/partials/cubeDesigner/dimensions.html
+++ b/webapp/app/partials/cubeDesigner/dimensions.html
@@ -240,10 +240,10 @@
 {{table}}[FactTable]
 
   
-Select All
+Select All
 Name
 Columns
-
+
   
   
 
@@ -266,10 +266,10 @@
 {{table}}[LookupTable]
 
   
-Select All
+Select All
 Name
 Columns
-
+
   
   
 
@@ -299,8 +299,8 @@
 
 
 
-OK
 Cancel
+OK
 
 
 

[04/33] kylin git commit: port KYLIN-2012 to new interface introduced in KYLIN-2125

[shaofengshi]

port KYLIN-2012 to new interface introduced in KYLIN-2125


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/cd2a06a5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/cd2a06a5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/cd2a06a5

Branch: refs/heads/v1.6.0-rc1-cdh5.7
Commit: cd2a06a5d373bdd5cfa90e78649d42e891711c43
Parents: 553d7c5
Author: Hongbin Ma 
Authored: Wed Oct 26 14:04:56 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Oct 27 08:34:11 2016 +0800

--
 .../java/org/apache/kylin/job/DeployUtil.java   |  5 ++-
 .../kylin/source/hive/BeelineHiveClient.java|  2 +-
 .../source/hive/HiveSourceTableLoader.java  | 32 +++
 .../apache/kylin/source/hive/SchemaChecker.java | 41 
 4 files changed, 36 insertions(+), 44 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/cd2a06a5/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
--
diff --git a/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java 
b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
index 4a24ad2..54feb24 100644
--- a/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
+++ b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
@@ -45,8 +45,9 @@ import org.apache.kylin.metadata.model.ColumnDesc;
 import org.apache.kylin.metadata.model.TableDesc;
 import org.apache.kylin.metadata.model.TableRef;
 import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.source.hive.HiveClient;
+import org.apache.kylin.source.hive.HiveClientFactory;
 import org.apache.kylin.source.hive.HiveCmdBuilder;
+import org.apache.kylin.source.hive.IHiveClient;
 import org.apache.kylin.source.kafka.TimedJsonStreamParser;
 import org.apache.maven.model.Model;
 import org.apache.maven.model.io.xpp3.MavenXpp3Reader;
@@ -234,7 +235,7 @@ public class DeployUtil {
 String tableFileDir = temp.getParent();
 temp.delete();
 
-HiveClient hiveClient = new HiveClient();
+IHiveClient hiveClient = HiveClientFactory.getHiveClient();
 // create hive tables
 hiveClient.executeHQL("CREATE DATABASE IF NOT EXISTS EDW");
 
hiveClient.executeHQL(generateCreateTableHql(metaMgr.getTableDesc(TABLE_CAL_DT.toUpperCase(;

http://git-wip-us.apache.org/repos/asf/kylin/blob/cd2a06a5/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
--
diff --git 
a/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java 
b/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
index c8d56a5..a84aeb1 100644
--- 
a/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
+++ 
b/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
@@ -207,7 +207,7 @@ public class BeelineHiveClient implements IHiveClient {
 
 BeelineHiveClient loader = new BeelineHiveClient("-n root --hiveconf 
hive.security.authorization.sqlstd.confwhitelist.append='mapreduce.job.*|dfs.*' 
-u 'jdbc:hive2://sandbox:1'");
 //BeelineHiveClient loader = new 
BeelineHiveClient(StringUtils.join(args, " "));
-HiveTableMeta hiveTableMeta = loader.getHiveTableMeta("default", 
"test001");
+HiveTableMeta hiveTableMeta = loader.getHiveTableMeta("default", 
"test_kylin_fact_part");
 System.out.println(hiveTableMeta);
 loader.close();
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/cd2a06a5/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java
--
diff --git 
a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java
 
b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java
index 388e72b..401e720 100644
--- 
a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java
+++ 
b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java
@@ -26,6 +26,7 @@ import java.util.Set;
 import java.util.UUID;
 
 import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.cube.CubeManager;
 import org.apache.kylin.engine.mr.HadoopUtil;
 import org.apache.kylin.metadata.MetadataConstants;
 import org.apache.kylin.metadata.MetadataManager;
@@ -34,8 +35,10 @@ import org.apache.kylin.metadata.model.TableDesc;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.google.common.collect.LinkedHashMultimap;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
+import com.google.common.collect.SetMultimap;
 

[09/33] kylin git commit: KYLIN-2137 Failed to run mr job when user put a kafka jar in hive's lib folder

[shaofengshi]

KYLIN-2137 Failed to run mr job when user put a kafka jar in hive's lib folder

Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e419681e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e419681e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e419681e

Branch: refs/heads/v1.6.0-rc1-cdh5.7
Commit: e419681e8c9c6135af40dcdc2b9b3b367b3e2456
Parents: 46a894d
Author: shaofengshi 
Authored: Fri Oct 28 18:15:33 2016 +0800
Committer: shaofengshi 
Committed: Fri Oct 28 22:11:33 2016 +0800

--
 .../apache/kylin/common/util/StringUtil.java| 13 +
 .../engine/mr/common/AbstractHadoopJob.java | 29 ++--
 2 files changed, 22 insertions(+), 20 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/e419681e/core-common/src/main/java/org/apache/kylin/common/util/StringUtil.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/util/StringUtil.java 
b/core-common/src/main/java/org/apache/kylin/common/util/StringUtil.java
index 15872aa..b336e4b 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/StringUtil.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/StringUtil.java
@@ -124,4 +124,17 @@ public class StringUtil {
 return result;
 }
 
+public static void appendWithSeparator(StringBuilder src, String append) {
+if (src == null) {
+throw new IllegalArgumentException();
+}
+if (src.length() > 0 && src.toString().endsWith(",") == false) {
+src.append(",");
+}
+
+if (StringUtils.isBlank(append) == false) {
+src.append(append);
+}
+}
+
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/e419681e/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
index a5b2d2e..417d63d 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
@@ -61,6 +61,7 @@ import org.apache.kylin.common.persistence.ResourceStore;
 import org.apache.kylin.common.util.CliCommandExecutor;
 import org.apache.kylin.common.util.OptionsHelper;
 import org.apache.kylin.common.util.StringSplitter;
+import org.apache.kylin.common.util.StringUtil;
 import org.apache.kylin.cube.CubeInstance;
 import org.apache.kylin.cube.CubeSegment;
 import org.apache.kylin.engine.mr.HadoopUtil;
@@ -202,24 +203,23 @@ public abstract class AbstractHadoopJob extends 
Configured implements Tool {
 String filteredHive = 
filterKylinHiveDependency(kylinHiveDependency);
 logger.info("Hive Dependencies After Filtered: " + filteredHive);
 
-if (kylinDependency.length() > 0)
-kylinDependency.append(",");
-kylinDependency.append(filteredHive);
+StringUtil.appendWithSeparator(kylinDependency, filteredHive);
 } else {
 
 logger.info("No hive dependency jars set in the environment, will 
find them from jvm:");
 
 try {
 String hiveExecJarPath = 
ClassUtil.findContainingJar(Class.forName("org.apache.hadoop.hive.ql.Driver"));
-kylinDependency.append(hiveExecJarPath).append(",");
+
+StringUtil.appendWithSeparator(kylinDependency, 
hiveExecJarPath);
 logger.info("hive-exec jar file: " + hiveExecJarPath);
 
 String hiveHCatJarPath = 
ClassUtil.findContainingJar(Class.forName("org.apache.hive.hcatalog.mapreduce.HCatInputFormat"));
-kylinDependency.append(hiveHCatJarPath).append(",");
+StringUtil.appendWithSeparator(kylinDependency, 
hiveHCatJarPath);
 logger.info("hive-catalog jar file: " + hiveHCatJarPath);
 
 String hiveMetaStoreJarPath = 
ClassUtil.findContainingJar(Class.forName("org.apache.hadoop.hive.metastore.api.Table"));
-kylinDependency.append(hiveMetaStoreJarPath).append(",");
+StringUtil.appendWithSeparator(kylinDependency, 
hiveMetaStoreJarPath);
 logger.info("hive-metastore jar file: " + 
hiveMetaStoreJarPath);
 } catch (ClassNotFoundException e) {
 logger.error("Cannot found hive dependency jars: " + e);
@@ -229,19 +229,13 @@ public abstract class AbstractHadoopJob extends 
Configured implements Tool {
 // for kafka dependencies

[02/33] kylin git commit: port KYLIN-2068 to BeelineHiveClient(KYLIN-2125)

[shaofengshi]

port KYLIN-2068 to BeelineHiveClient(KYLIN-2125)


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/553d7c56
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/553d7c56
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/553d7c56

Branch: refs/heads/v1.6.0-rc1-cdh5.7
Commit: 553d7c56a0af49cb1c003f94543b4e600cd6f66e
Parents: 9ed775c
Author: Hongbin Ma 
Authored: Wed Oct 26 13:37:03 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Oct 27 08:34:11 2016 +0800

--
 .../org/apache/kylin/source/hive/BeelineHiveClient.java   | 10 +-
 .../java/org/apache/kylin/source/hive/CLIHiveClient.java  |  4 ++--
 .../apache/kylin/source/hive/HiveSourceTableLoader.java   |  1 +
 .../java/org/apache/kylin/source/hive/HiveTableMeta.java  |  6 --
 4 files changed, 12 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/553d7c56/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
--
diff --git 
a/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java 
b/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
index 0fbc39b..c8d56a5 100644
--- 
a/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
+++ 
b/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
@@ -115,7 +115,7 @@ public class BeelineHiveClient implements IHiveClient {
 
 List allColumns = 
Lists.newArrayList();
 while (columns.next()) {
-allColumns.add(new 
HiveTableMeta.HiveTableColumnMeta(columns.getString(4), columns.getString(6)));
+allColumns.add(new 
HiveTableMeta.HiveTableColumnMeta(columns.getString(4), columns.getString(6), 
columns.getString(12)));
 }
 builder.setAllColumns(allColumns);
 
@@ -139,7 +139,7 @@ public class BeelineHiveClient implements IHiveClient {
 if ("".equals(resultSet.getString(1).trim())) {
 break;
 }
-partitionColumns.add(new 
HiveTableMeta.HiveTableColumnMeta(resultSet.getString(1).trim(), 
resultSet.getString(2).trim()));
+partitionColumns.add(new 
HiveTableMeta.HiveTableColumnMeta(resultSet.getString(1).trim(), 
resultSet.getString(2).trim(), resultSet.getString(3).trim()));
 }
 builder.setPartitionColumns(partitionColumns);
 }
@@ -205,9 +205,9 @@ public class BeelineHiveClient implements IHiveClient {
 
 public static void main(String[] args) throws SQLException {
 
-//BeelineHiveClient loader = new BeelineHiveClient("-n root --hiveconf 
hive.security.authorization.sqlstd.confwhitelist.append='mapreduce.job.*|dfs.*' 
-u 'jdbc:hive2://sandbox:1'");
-BeelineHiveClient loader = new 
BeelineHiveClient(StringUtils.join(args, " "));
-HiveTableMeta hiveTableMeta = loader.getHiveTableMeta("default", 
"events");
+BeelineHiveClient loader = new BeelineHiveClient("-n root --hiveconf 
hive.security.authorization.sqlstd.confwhitelist.append='mapreduce.job.*|dfs.*' 
-u 'jdbc:hive2://sandbox:1'");
+//BeelineHiveClient loader = new 
BeelineHiveClient(StringUtils.join(args, " "));
+HiveTableMeta hiveTableMeta = loader.getHiveTableMeta("default", 
"test001");
 System.out.println(hiveTableMeta);
 loader.close();
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/553d7c56/source-hive/src/main/java/org/apache/kylin/source/hive/CLIHiveClient.java
--
diff --git 
a/source-hive/src/main/java/org/apache/kylin/source/hive/CLIHiveClient.java 
b/source-hive/src/main/java/org/apache/kylin/source/hive/CLIHiveClient.java
index ea74470..60cf47a 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/CLIHiveClient.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/CLIHiveClient.java
@@ -89,11 +89,11 @@ public class CLIHiveClient implements IHiveClient {
 List allColumns = 
Lists.newArrayList();
 List partitionColumns = 
Lists.newArrayList();
 for (FieldSchema fieldSchema : allFields) {
-allColumns.add(new 
HiveTableMeta.HiveTableColumnMeta(fieldSchema.getName(), 
fieldSchema.getType()));
+allColumns.add(new 
HiveTableMeta.HiveTableColumnMeta(fieldSchema.getName(), fieldSchema.getType(), 
fieldSchema.getComment()));
 }
 if (partitionFields != null && partitionFields.size() > 0) {
 for (FieldSchema fieldSchema : partitionFields) {
-partitionColumns.add(new 
HiveTableMeta.HiveTableColumnMeta(fieldSchema.getName(), 

[24/33] kylin git commit: KYLIN-2114 remove builder dictionary config to cubeConfig

[shaofengshi]

KYLIN-2114 remove builder dictionary config to cubeConfig


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/eaaaf2f7
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/eaaaf2f7
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/eaaaf2f7

Branch: refs/heads/v1.6.0-rc1-cdh5.7
Commit: eaaaf2f7bbe3ec617cd827eeeac31cdad8e8950e
Parents: c780e43
Author: Jason 
Authored: Tue Nov 1 19:15:48 2016 +0800
Committer: Jason 
Committed: Tue Nov 1 19:16:21 2016 +0800

--
 webapp/app/js/model/cubeConfig.js   | 5 -
 webapp/app/partials/cubeDesigner/advanced_settings.html | 2 +-
 2 files changed, 5 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/eaaaf2f7/webapp/app/js/model/cubeConfig.js
--
diff --git a/webapp/app/js/model/cubeConfig.js 
b/webapp/app/js/model/cubeConfig.js
index a77ab4f..87413dc 100644
--- a/webapp/app/js/model/cubeConfig.js
+++ b/webapp/app/js/model/cubeConfig.js
@@ -104,5 +104,8 @@ KylinApp.constant('cubeConfig', {
   rowKeyShardOptions:[
 true,false
   ],
-  statusNeedNofity:['ERROR', 'DISCARDED', 'SUCCEED']
+  statusNeedNofity:['ERROR', 'DISCARDED', 'SUCCEED'],
+  buildDictionaries:[
+{name:"Global Dictionary", 
value:"org.apache.kylin.dict.GlobalDictionaryBuilder"}
+  ]
 });

http://git-wip-us.apache.org/repos/asf/kylin/blob/eaaaf2f7/webapp/app/partials/cubeDesigner/advanced_settings.html
--
diff --git a/webapp/app/partials/cubeDesigner/advanced_settings.html 
b/webapp/app/partials/cubeDesigner/advanced_settings.html
index a1e3c80..0a9dc48 100755
--- a/webapp/app/partials/cubeDesigner/advanced_settings.html
+++ b/webapp/app/partials/cubeDesigner/advanced_settings.html
@@ -370,8 +370,8 @@
  

- Global Dictionary
  -- Select a builder class--

  

[28/33] kylin git commit: KYLIN-2122 fix class visibility error

[shaofengshi]

KYLIN-2122 fix class visibility error


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4a689b2d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4a689b2d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4a689b2d

Branch: refs/heads/v1.6.0-rc1-cdh5.7
Commit: 4a689b2db039aaa0dcc9cb3ce50192f11961b8a4
Parents: 2beccbf
Author: shaofengshi 
Authored: Wed Nov 2 13:58:29 2016 +0800
Committer: shaofengshi 
Committed: Wed Nov 2 13:58:29 2016 +0800

--
 .../src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/4a689b2d/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java
--
diff --git 
a/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java 
b/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java
index cdd7272..99ebb6a 100644
--- a/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java
+++ b/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java
@@ -208,7 +208,7 @@ public class KafkaMRInput implements IMRInput {
 }
 }
 
-static class GarbageCollectionStep extends AbstractExecutable {
+public static class GarbageCollectionStep extends AbstractExecutable {
 private static final Logger logger = 
LoggerFactory.getLogger(GarbageCollectionStep.class);
 
 @Override

[30/33] kylin git commit: view cube bug

[shaofengshi]

view cube bug

Signed-off-by: shaofengshi 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7a379305
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7a379305
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7a379305

Branch: refs/heads/v1.6.0-rc1-cdh5.7
Commit: 7a3793059b5e790ba0de667b7feaabae7a468119
Parents: 7d13664
Author: luguosheng <550175...@qq.com>
Authored: Wed Nov 2 16:17:15 2016 +0800
Committer: shaofengshi 
Committed: Wed Nov 2 16:26:58 2016 +0800

--
 webapp/app/js/controllers/cubeEdit.js | 8 
 1 file changed, 4 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/7a379305/webapp/app/js/controllers/cubeEdit.js
--
diff --git a/webapp/app/js/controllers/cubeEdit.js 
b/webapp/app/js/controllers/cubeEdit.js
index 09dae15..acd4d32 100755
--- a/webapp/app/js/controllers/cubeEdit.js
+++ b/webapp/app/js/controllers/cubeEdit.js
@@ -293,11 +293,11 @@ KylinApp.controller('CubeEditCtrl', function ($scope, $q, 
$routeParams, $locatio
 });
 
 var queryParam = {
-  cube_name: $routeParams.cubeName
+  cubeId: $routeParams.cubeName
 };
-CubeService.list(queryParam, {},function(instance){
-  if (instance.length > 0) {
-$scope.instance = instance[0];
+CubeService.getCube(queryParam, {},function(instance){
+  if (instance) {
+$scope.instance = instance;
 $scope.state.cubeInstance =angular.toJson($scope.instance,true);
 
   } else {

[25/33] kylin git commit: KYLIN-2024 kylin TopN only support the first measure

[shaofengshi]

KYLIN-2024 kylin TopN only support the first measure


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1d8e8e53
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1d8e8e53
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1d8e8e53

Branch: refs/heads/v1.6.0-rc1-cdh5.7
Commit: 1d8e8e5389dae3960d9540dea68321a0f8b0cfb8
Parents: eaaaf2f
Author: shaofengshi 
Authored: Wed Nov 2 13:20:10 2016 +0800
Committer: shaofengshi 
Committed: Wed Nov 2 13:42:06 2016 +0800

--
 .../kylin/measure/topn/TopNMeasureType.java  | 19 +--
 1 file changed, 13 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/1d8e8e53/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
 
b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
index 8c98672..7e788e4 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
@@ -39,8 +39,8 @@ import org.apache.kylin.metadata.datatype.DataTypeSerializer;
 import org.apache.kylin.metadata.model.FunctionDesc;
 import org.apache.kylin.metadata.model.MeasureDesc;
 import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.metadata.realization.SQLDigest;
 import 
org.apache.kylin.metadata.realization.CapabilityResult.CapabilityInfluence;
+import org.apache.kylin.metadata.realization.SQLDigest;
 import org.apache.kylin.metadata.tuple.Tuple;
 import org.apache.kylin.metadata.tuple.TupleInfo;
 import org.slf4j.Logger;
@@ -307,15 +307,16 @@ public class TopNMeasureType extends 
MeasureType {
 
 @Override
 public void adjustSqlDigest(List measureDescs, SQLDigest 
sqlDigest) {
+if (sqlDigest.aggregations.size() > 1) {
+return;
+}
+
 for (MeasureDesc measureDesc : measureDescs) {
 FunctionDesc topnFunc = measureDesc.getFunction();
 List topnLiteralCol = getTopNLiteralColumn(topnFunc);
 
-if (sqlDigest.groupbyColumns.containsAll(topnLiteralCol) == false)
-return;
-
-if (sqlDigest.aggregations.size() > 1) {
-return;
+if (sqlDigest.groupbyColumns.containsAll(topnLiteralCol) == false) 
{
+continue;
 }
 
 if (sqlDigest.aggregations.size() > 0) {
@@ -324,12 +325,18 @@ public class TopNMeasureType extends 
MeasureType {
 logger.warn("When query with topN, only SUM/Count function 
is allowed.");
 return;
 }
+
+if (isTopNCompatibleSum(measureDesc.getFunction(), origFunc) 
== false) {
+continue;
+}
+
 logger.info("Rewrite function " + origFunc + " to " + 
topnFunc);
 }
 
 sqlDigest.aggregations = Lists.newArrayList(topnFunc);
 sqlDigest.groupbyColumns.removeAll(topnLiteralCol);
 sqlDigest.metricColumns.addAll(topnLiteralCol);
+break;
 }
 }
 

[17/33] kylin git commit: KYLIN Add derived checkbox for lookup table columns on Auto Generate Dimensions panel

[shaofengshi]

KYLIN Add derived checkbox for lookup table columns on Auto Generate Dimensions 
panel

Signed-off-by: Jason 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d01a1f74
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d01a1f74
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d01a1f74

Branch: refs/heads/v1.6.0-rc1-cdh5.7
Commit: d01a1f7433a2529c5075f09786a50a86d5ea4653
Parents: 5218c9f
Author: chenzhx <346839...@qq.com>
Authored: Fri Oct 28 15:45:22 2016 +0800
Committer: Jason 
Committed: Mon Oct 31 17:09:58 2016 +0800

--
 webapp/app/js/controllers/cubeDimensions.js | 110 +++
 .../app/partials/cubeDesigner/dimensions.html   |  26 ++---
 2 files changed, 100 insertions(+), 36 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/d01a1f74/webapp/app/js/controllers/cubeDimensions.js
--
diff --git a/webapp/app/js/controllers/cubeDimensions.js 
b/webapp/app/js/controllers/cubeDimensions.js
index ba71c96..7cb850b 100644
--- a/webapp/app/js/controllers/cubeDimensions.js
+++ b/webapp/app/js/controllers/cubeDimensions.js
@@ -74,11 +74,12 @@ KylinApp.controller('CubeDimensionsCtrl', function ($scope, 
$modal,MetaModel,cub
 cols[i].isLookup = false;
 
 // Default not selected and not disabled.
-factSelectAvailable[cols[i].name] = {name:cols[i].name ,selected: 
false, disabled: false};
+factSelectAvailable[cols[i].name] = {name:cols[i].name ,selected: 
false};
 
 }
 
 $scope.availableColumns[factTable] = cols;
+factSelectAvailable.all=false;
 $scope.selectedColumns[factTable] = factSelectAvailable;
 $scope.availableTables.push(factTable);
 
@@ -96,10 +97,11 @@ KylinApp.controller('CubeDimensionsCtrl', function ($scope, 
$modal,MetaModel,cub
 cols2[k].isLookup = true;
 
 // Default not selected and not disabled.
-lookupSelectAvailable[cols2[k].name] = 
{name:cols2[k].name,selected: false, disabled: false};
+lookupSelectAvailable[cols2[k].name] = 
{name:cols2[k].name,selected: false};
 }
 
 $scope.availableColumns[lookups[j].table] = cols2;
+lookupSelectAvailable.all=false;
 $scope.selectedColumns[lookups[j].table] = lookupSelectAvailable;
 if($scope.availableTables.indexOf(lookups[j].table)==-1){
 $scope.availableTables.push(lookups[j].table);
@@ -111,14 +113,22 @@ KylinApp.controller('CubeDimensionsCtrl', function 
($scope, $modal,MetaModel,cub
 $scope.initColumnStatus = function () {
 angular.forEach($scope.cubeMetaFrame.dimensions, function (dim) {
 var cols = dimCols(dim);
-
 angular.forEach(cols, function (colName) {
   if(dim.derived){
- $scope.selectedColumns[dim.table][colName] = {name:dim.name, 
selected: true, disabled: true,normal:"false"};
+ $scope.selectedColumns[dim.table][colName] = {name:dim.name, 
selected: true, normal:"false"};
   }else{
- $scope.selectedColumns[dim.table][colName] = {name:dim.name, 
selected: true, disabled: true,normal:"true"};
+ $scope.selectedColumns[dim.table][colName] = {name:dim.name, 
selected: true, normal:"true"};
   }
-   });
+});
+});
+angular.forEach($scope.selectedColumns,function(value,table){
+  var all=true;
+  angular.forEach(value,function(col){
+   if(col.selected==false& col=="object"){
+all=false;
+   }
+ });
+ $scope.selectedColumns[table].all=all;
 });
 };
 
@@ -300,9 +310,9 @@ KylinApp.controller('CubeDimensionsCtrl', function ($scope, 
$modal,MetaModel,cub
 var cols = dimCols(dim);
 angular.forEach(cols, function (colName) {
 if(dim.table==$scope.metaModel.model.fact_table){
-   $scope.selectedColumns[dim.table][colName] = 
{name:colName,selected: false, disabled: false};
+   $scope.selectedColumns[dim.table][colName] = 
{name:colName,selected: false};
 }else{
-   $scope.selectedColumns[dim.table][colName] = 
{name:colName,selected: false, disabled: false};
+   $scope.selectedColumns[dim.table][colName] = 
{name:colName,selected: false};
 }
 });
 };
@@ -395,25 +405,77 @@ KylinApp.controller('CubeDimensionsCtrl', function 
($scope, $modal,MetaModel,cub
 };
 
 $scope.autoChange= function(table,name){
-if($scope.selectedColumns[table][name].selected==false){
- 

[19/33] kylin git commit: AbstractHadoopJob: set hive dependency filter as a config entry

[shaofengshi]

AbstractHadoopJob: set hive dependency filter as a config entry


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a8922e87
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a8922e87
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a8922e87

Branch: refs/heads/v1.6.0-rc1-cdh5.7
Commit: a8922e87b1f483dc010185f4a08aae970898c45d
Parents: 21ecfce
Author: Hongbin Ma 
Authored: Mon Oct 31 17:03:42 2016 +0800
Committer: Hongbin Ma 
Committed: Mon Oct 31 17:11:01 2016 +0800

--
 .../main/java/org/apache/kylin/common/KylinConfigBase.java  | 8 ++--
 .../apache/kylin/engine/mr/common/AbstractHadoopJob.java| 9 +++--
 2 files changed, 9 insertions(+), 8 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/a8922e87/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 26c280b..011155f 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -22,9 +22,9 @@ import java.io.File;
 import java.io.IOException;
 import java.io.Serializable;
 import java.util.Map;
+import java.util.Map.Entry;
 import java.util.Properties;
 import java.util.SortedSet;
-import java.util.Map.Entry;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
@@ -504,7 +504,7 @@ abstract public class KylinConfigBase implements 
Serializable {
 public boolean getBadQueryPersistentEnabled() {
 return 
Boolean.parseBoolean(getOptional("kylin.query.badquery.persistent.enable", 
"true"));
 }
-
+
 public String[] getQueryTransformers() {
 return getOptionalStringArray("kylin.query.transformers", new 
String[0]);
 }
@@ -633,6 +633,10 @@ abstract public class KylinConfigBase implements 
Serializable {
 return 
this.getOptional("kylin.job.hive.database.for.intermediatetable", "default");
 }
 
+public String getHiveDependencyFilterList() {
+return this.getOptional("kylin.job.dependency.filterlist", 
"[^,]*hive-exec[0-9.-]+[^,]*?\\.jar" + "|" + 
"[^,]*hive-metastore[0-9.-]+[^,]*?\\.jar" + "|" + 
"[^,]*hive-hcatalog-core[0-9.-]+[^,]*?\\.jar");
+}
+
 public String getKylinOwner() {
 return this.getOptional("kylin.owner", "");
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/a8922e87/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
index 72f4437..77791ce 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
@@ -96,8 +96,6 @@ public abstract class AbstractHadoopJob extends Configured 
implements Tool {
 
 private static final String MAP_REDUCE_CLASSPATH = 
"mapreduce.application.classpath";
 
-private static final String KYLIN_HIVE_DEPENDENCY_JARS = 
"[^,]*hive-exec[0-9.-]+[^,]*?\\.jar" + "|" + 
"[^,]*hive-metastore[0-9.-]+[^,]*?\\.jar" + "|" + 
"[^,]*hive-hcatalog-core[0-9.-]+[^,]*?\\.jar";
-
 protected static void runJob(Tool job, String[] args) {
 try {
 int exitCode = ToolRunner.run(job, args);
@@ -200,8 +198,7 @@ public abstract class AbstractHadoopJob extends Configured 
implements Tool {
 kylinHiveDependency = kylinHiveDependency.replace(":", ",");
 
 logger.info("Hive Dependencies Before Filtered: " + 
kylinHiveDependency);
-//String filteredHive = 
filterKylinHiveDependency(kylinHiveDependency);
-String filteredHive = kylinHiveDependency;
+String filteredHive = 
filterKylinHiveDependency(kylinHiveDependency, kylinConf);
 logger.info("Hive Dependencies After Filtered: " + filteredHive);
 
 StringUtil.appendWithSeparator(kylinDependency, filteredHive);
@@ -259,13 +256,13 @@ public abstract class AbstractHadoopJob extends 
Configured implements Tool {
 }
 }
 
-private String filterKylinHiveDependency(String kylinHiveDependency) {
+private String filterKylinHiveDependency(String kylinHiveDependency, 
KylinConfig config) {
 if (StringUtils.isBlank(kylinHiveDependency))
 return "";
 
 StringBuilder jarList = new StringBuilder();
 
-Pattern hivePattern = 

[21/33] kylin git commit: KYLIN-2143 allow more options from Extended Columns, COUNT_DISTINCT, RAW_TABLE

[shaofengshi]

KYLIN-2143 allow more options from Extended Columns,COUNT_DISTINCT,RAW_TABLE


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/67428a5d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/67428a5d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/67428a5d

Branch: refs/heads/v1.6.0-rc1-cdh5.7
Commit: 67428a5d83c6604b96d62da08edf48f844f02437
Parents: 8172d0b
Author: Jason 
Authored: Tue Nov 1 16:51:51 2016 +0800
Committer: Jason 
Committed: Tue Nov 1 16:52:31 2016 +0800

--
 webapp/app/js/controllers/cubeEdit.js  | 34 +++-
 webapp/app/partials/cubeDesigner/measures.html | 96 +++--
 2 files changed, 82 insertions(+), 48 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/67428a5d/webapp/app/js/controllers/cubeEdit.js
--
diff --git a/webapp/app/js/controllers/cubeEdit.js 
b/webapp/app/js/controllers/cubeEdit.js
index 5f46799..06d3120 100755
--- a/webapp/app/js/controllers/cubeEdit.js
+++ b/webapp/app/js/controllers/cubeEdit.js
@@ -137,18 +137,44 @@ KylinApp.controller('CubeEditCtrl', function ($scope, $q, 
$routeParams, $locatio
 return me_columns;
   };
 
-  $scope.getGroupByColumns = function () {
-var groupBy_columns=[];
+  $scope.getAllModelDimMeasureColumns = function () {
+var me_columns = [];
+if($scope.metaModel.model.metrics){
+  angular.forEach($scope.metaModel.model.metrics,function(metric,index){
+me_columns.push(metric);
+  })
+}
+
+
angular.forEach($scope.metaModel.model.dimensions,function(dimension,index){
+  if(dimension.columns){
+me_columns = me_columns.concat(dimension.columns);
+  }
+})
 
+return distinct_array(me_columns);
+  };
+
+  $scope.getAllModelDimColumns = function () {
+var me_columns = [];
 
angular.forEach($scope.metaModel.model.dimensions,function(dimension,index){
   if(dimension.columns){
-groupBy_columns = groupBy_columns.concat(dimension.columns);
+me_columns = me_columns.concat(dimension.columns);
   }
 })
 
-return groupBy_columns;
+return distinct_array(me_columns);
   };
 
+  function distinct_array(arrays){
+var arr = [];
+for(var item in arrays){
+  if(arr.indexOf(arrays[item])==-1){
+arr.push(arrays[item]);
+  }
+}
+return arr;
+  }
+
 
   $scope.getExtendedHostColumn = function(){
 var me_columns = [];

http://git-wip-us.apache.org/repos/asf/kylin/blob/67428a5d/webapp/app/partials/cubeDesigner/measures.html
--
diff --git a/webapp/app/partials/cubeDesigner/measures.html 
b/webapp/app/partials/cubeDesigner/measures.html
index b4513f2..c85bfd2 100755
--- a/webapp/app/partials/cubeDesigner/measures.html
+++ b/webapp/app/partials/cubeDesigner/measures.html
@@ -149,15 +149,22 @@
   
ng-init="newMeasure.function.parameter.value = 1">1
 
 
+ng-options="column as column for 
column in getAllModelDimMeasureColumns(newMeasure)" >
+  -- Select a Column 
--
+
+
   -- Select a Column 
--
 
 
   -- Select a Column 
--
@@ -165,44 +172,6 @@
   
   
   
-  
-  
-  
-  
-Return Type
-Maximum length of extended column  
-  
-  
-  
-  
-  
-
-  
-
-
-
-
-  {{newMeasure.function.returntype | uppercase}}
-  
-  
-  
-  
-
 
   
 
@@ -212,12 +181,51 @@
   
 
+ng-options="column as column for column in 
getAllModelDimColumns()" >
+  -- Select a Column --
+
+  
+
+  
+
+   

[06/33] kylin git commit: KYLIN-2134 dont reuse Object[] returned to calcite

[shaofengshi]

KYLIN-2134 dont reuse Object[] returned to calcite


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8199a90c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8199a90c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8199a90c

Branch: refs/heads/v1.6.0-rc1-cdh5.7
Commit: 8199a90c57df12e8ad4a19737ffa9aa17a69f3eb
Parents: 1af3d9e
Author: Li Yang 
Authored: Thu Oct 27 17:09:22 2016 +0800
Committer: Hongbin Ma 
Committed: Fri Oct 28 09:51:51 2016 +0800

--
 .../kylin/storage/gtrecord/SegmentCubeTupleIterator.java | 4 
 .../org/apache/kylin/query/enumerator/OLAPEnumerator.java| 8 
 2 files changed, 4 insertions(+), 8 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/8199a90c/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SegmentCubeTupleIterator.java
--
diff --git 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SegmentCubeTupleIterator.java
 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SegmentCubeTupleIterator.java
index 61267ae..00ba247 100644
--- 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SegmentCubeTupleIterator.java
+++ 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SegmentCubeTupleIterator.java
@@ -37,8 +37,6 @@ import org.apache.kylin.storage.StorageContext;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.google.common.base.Preconditions;
-
 public class SegmentCubeTupleIterator implements ITupleIterator {
 
 private static final Logger logger = 
LoggerFactory.getLogger(SegmentCubeTupleIterator.class);
@@ -98,8 +96,6 @@ public class SegmentCubeTupleIterator implements 
ITupleIterator {
 }
 GTRecord curRecord = gtItr.next();
 
-Preconditions.checkNotNull(cubeTupleConverter);
-
 // translate into tuple
 advMeasureFillers = cubeTupleConverter.translateResult(curRecord, 
tuple);
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/8199a90c/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
--
diff --git 
a/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java 
b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
index dafbb5f..c7b3c71 100644
--- a/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
+++ b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
@@ -18,6 +18,7 @@
 
 package org.apache.kylin.query.enumerator;
 
+import java.util.Arrays;
 import java.util.Map;
 import java.util.Properties;
 
@@ -77,10 +78,9 @@ public class OLAPEnumerator implements Enumerator {
 }
 
 private Object[] convertCurrentRow(ITuple tuple) {
-// make sure the tuple layout is correct
-//assert 
tuple.getAllFields().equals(olapContext.returnTupleInfo.getAllFields());
-
-current = tuple.getAllValues();
+// give calcite a new array every time, see details in KYLIN-2134
+Object[] values = tuple.getAllValues();
+current = Arrays.copyOf(values, values.length);
 return current;
 }
 

[26/33] kylin git commit: KYLIN-2147 Move the creation of HTable after cube be built

[shaofengshi]

KYLIN-2147 Move the creation of HTable after cube be built

Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b6f608f7
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b6f608f7
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b6f608f7

Branch: refs/heads/v1.6.0-rc1-cdh5.7
Commit: b6f608f7e145b32f2c9c50bb2b719b40c1c2eb8b
Parents: 1d8e8e5
Author: shaofengshi 
Authored: Wed Nov 2 13:39:26 2016 +0800
Committer: shaofengshi 
Committed: Wed Nov 2 13:42:14 2016 +0800

--
 .../apache/kylin/storage/hbase/steps/HBaseMROutput2Transition.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/b6f608f7/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2Transition.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2Transition.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2Transition.java
index c4df354..a063efe 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2Transition.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2Transition.java
@@ -49,11 +49,11 @@ public class HBaseMROutput2Transition implements IMROutput2 
{
 
 @Override
 public void addStepPhase2_BuildDictionary(DefaultChainedExecutable 
jobFlow) {
-
jobFlow.addTask(steps.createCreateHTableStepWithStats(jobFlow.getId()));
 }
 
 @Override
 public void addStepPhase3_BuildCube(DefaultChainedExecutable 
jobFlow) {
+
jobFlow.addTask(steps.createCreateHTableStepWithStats(jobFlow.getId()));
 
jobFlow.addTask(steps.createConvertCuboidToHfileStep(jobFlow.getId()));
 jobFlow.addTask(steps.createBulkLoadStep(jobFlow.getId()));
 }

[14/33] kylin git commit: KYLIN-2116 Remove field delimiter in hive flatten table ddl

[shaofengshi]

KYLIN-2116 Remove field delimiter in hive flatten table ddl


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/82e99078
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/82e99078
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/82e99078

Branch: refs/heads/v1.6.0-rc1-cdh5.7
Commit: 82e99078c08505b426d34ad6395a14b3d6244b07
Parents: eabbd4e
Author: lidongsjtu 
Authored: Wed Oct 26 22:49:19 2016 +0800
Committer: lidongsjtu 
Committed: Mon Oct 31 13:46:36 2016 +0800

--
 core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java | 2 --
 1 file changed, 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/82e99078/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
--
diff --git a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java 
b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
index 79cf924..c5f38e7 100644
--- a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
+++ b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
@@ -92,8 +92,6 @@ public class JoinedFlatTable {
 ddl.append(colName(col.getCanonicalName()) + " " + 
getHiveDataType(col.getDatatype()) + "\n");
 }
 ddl.append(")" + "\n");
-
-ddl.append("ROW FORMAT DELIMITED FIELDS TERMINATED BY '\\177'" + "\n");
 ddl.append("STORED AS SEQUENCEFILE" + "\n");
 ddl.append("LOCATION '" + getTableDir(flatDesc, storageDfsDir) + 
"';").append("\n");
 // ddl.append("TBLPROPERTIES ('serialization.null.format'='N')" +

[01/33] kylin git commit: KYLIN-2127 UI bug fix for Extend Column [Forced Update!]

[shaofengshi]

Repository: kylin
Updated Branches:
  refs/heads/v1.6.0-rc1-cdh5.7 afa62b58f -> 56215d0f0 (forced update)


KYLIN-2127 UI bug fix for Extend Column

Signed-off-by: Jason 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e094b768
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e094b768
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e094b768

Branch: refs/heads/v1.6.0-rc1-cdh5.7
Commit: e094b76870bbc12ecf0a29e5ff23e4dfce7a0243
Parents: 772fa54
Author: kangkaisen 
Authored: Tue Oct 25 22:27:54 2016 +0800
Committer: Jason 
Committed: Wed Oct 26 18:47:02 2016 +0800

--
 webapp/app/js/controllers/cubeMeasures.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/e094b768/webapp/app/js/controllers/cubeMeasures.js
--
diff --git a/webapp/app/js/controllers/cubeMeasures.js 
b/webapp/app/js/controllers/cubeMeasures.js
index dcf26b7..938e918 100644
--- a/webapp/app/js/controllers/cubeMeasures.js
+++ b/webapp/app/js/controllers/cubeMeasures.js
@@ -229,7 +229,7 @@ KylinApp.controller('CubeMeasuresCtrl', function ($scope, 
$modal,MetaModel,cubes
 $scope.nextPara = {
   "type":"column",
   "value":"",
-  "next_parameter":{}
+  "next_parameter": null
 }
 if($scope.newMeasure){
   $scope.newMeasure.function.parameter.next_parameter = null;

[02/32] kylin git commit: port KYLIN-2012 to new interface introduced in KYLIN-2125

[shaofengshi]

port KYLIN-2012 to new interface introduced in KYLIN-2125


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/cd2a06a5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/cd2a06a5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/cd2a06a5

Branch: refs/heads/v1.6.0-rc1-hbase1.x
Commit: cd2a06a5d373bdd5cfa90e78649d42e891711c43
Parents: 553d7c5
Author: Hongbin Ma 
Authored: Wed Oct 26 14:04:56 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Oct 27 08:34:11 2016 +0800

--
 .../java/org/apache/kylin/job/DeployUtil.java   |  5 ++-
 .../kylin/source/hive/BeelineHiveClient.java|  2 +-
 .../source/hive/HiveSourceTableLoader.java  | 32 +++
 .../apache/kylin/source/hive/SchemaChecker.java | 41 
 4 files changed, 36 insertions(+), 44 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/cd2a06a5/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
--
diff --git a/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java 
b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
index 4a24ad2..54feb24 100644
--- a/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
+++ b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
@@ -45,8 +45,9 @@ import org.apache.kylin.metadata.model.ColumnDesc;
 import org.apache.kylin.metadata.model.TableDesc;
 import org.apache.kylin.metadata.model.TableRef;
 import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.source.hive.HiveClient;
+import org.apache.kylin.source.hive.HiveClientFactory;
 import org.apache.kylin.source.hive.HiveCmdBuilder;
+import org.apache.kylin.source.hive.IHiveClient;
 import org.apache.kylin.source.kafka.TimedJsonStreamParser;
 import org.apache.maven.model.Model;
 import org.apache.maven.model.io.xpp3.MavenXpp3Reader;
@@ -234,7 +235,7 @@ public class DeployUtil {
 String tableFileDir = temp.getParent();
 temp.delete();
 
-HiveClient hiveClient = new HiveClient();
+IHiveClient hiveClient = HiveClientFactory.getHiveClient();
 // create hive tables
 hiveClient.executeHQL("CREATE DATABASE IF NOT EXISTS EDW");
 
hiveClient.executeHQL(generateCreateTableHql(metaMgr.getTableDesc(TABLE_CAL_DT.toUpperCase(;

http://git-wip-us.apache.org/repos/asf/kylin/blob/cd2a06a5/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
--
diff --git 
a/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java 
b/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
index c8d56a5..a84aeb1 100644
--- 
a/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
+++ 
b/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
@@ -207,7 +207,7 @@ public class BeelineHiveClient implements IHiveClient {
 
 BeelineHiveClient loader = new BeelineHiveClient("-n root --hiveconf 
hive.security.authorization.sqlstd.confwhitelist.append='mapreduce.job.*|dfs.*' 
-u 'jdbc:hive2://sandbox:1'");
 //BeelineHiveClient loader = new 
BeelineHiveClient(StringUtils.join(args, " "));
-HiveTableMeta hiveTableMeta = loader.getHiveTableMeta("default", 
"test001");
+HiveTableMeta hiveTableMeta = loader.getHiveTableMeta("default", 
"test_kylin_fact_part");
 System.out.println(hiveTableMeta);
 loader.close();
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/cd2a06a5/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java
--
diff --git 
a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java
 
b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java
index 388e72b..401e720 100644
--- 
a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java
+++ 
b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java
@@ -26,6 +26,7 @@ import java.util.Set;
 import java.util.UUID;
 
 import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.cube.CubeManager;
 import org.apache.kylin.engine.mr.HadoopUtil;
 import org.apache.kylin.metadata.MetadataConstants;
 import org.apache.kylin.metadata.MetadataManager;
@@ -34,8 +35,10 @@ import org.apache.kylin.metadata.model.TableDesc;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.google.common.collect.LinkedHashMultimap;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
+import 

[20/32] kylin git commit: KYLIN-1726 apply kylin_job_conf.xml on KafkaFlatTableJob

[shaofengshi]

KYLIN-1726 apply kylin_job_conf.xml on KafkaFlatTableJob


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8172d0ba
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8172d0ba
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8172d0ba

Branch: refs/heads/v1.6.0-rc1-hbase1.x
Commit: 8172d0ba3e6768c803528b457db6cc3a464c8e14
Parents: a8922e8
Author: shaofengshi 
Authored: Tue Nov 1 15:27:52 2016 +0800
Committer: shaofengshi 
Committed: Tue Nov 1 15:27:52 2016 +0800

--
 .../org/apache/kylin/source/kafka/hadoop/KafkaFlatTableJob.java   | 3 +++
 1 file changed, 3 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/8172d0ba/source-kafka/src/main/java/org/apache/kylin/source/kafka/hadoop/KafkaFlatTableJob.java
--
diff --git 
a/source-kafka/src/main/java/org/apache/kylin/source/kafka/hadoop/KafkaFlatTableJob.java
 
b/source-kafka/src/main/java/org/apache/kylin/source/kafka/hadoop/KafkaFlatTableJob.java
index e20b20a..5fe6e00 100644
--- 
a/source-kafka/src/main/java/org/apache/kylin/source/kafka/hadoop/KafkaFlatTableJob.java
+++ 
b/source-kafka/src/main/java/org/apache/kylin/source/kafka/hadoop/KafkaFlatTableJob.java
@@ -18,6 +18,7 @@
 
 package org.apache.kylin.source.kafka.hadoop;
 
+import org.apache.kylin.job.engine.JobEngineConfig;
 import org.apache.kylin.source.kafka.util.KafkaClient;
 import org.apache.commons.cli.Options;
 import org.apache.hadoop.fs.Path;
@@ -97,6 +98,8 @@ public class KafkaFlatTableJob extends AbstractHadoopJob {
 throw new IllegalArgumentException("Invalid Kafka information, 
brokers " + brokers + ", topic " + topic);
 }
 
+JobEngineConfig jobEngineConfig = new 
JobEngineConfig(KylinConfig.getInstanceFromEnv());
+job.getConfiguration().addResource(new 
Path(jobEngineConfig.getHadoopJobConfFilePath(null)));
 job.getConfiguration().set(CONFIG_KAFKA_BROKERS, brokers);
 job.getConfiguration().set(CONFIG_KAFKA_TOPIC, topic);
 job.getConfiguration().set(CONFIG_KAFKA_TIMEOUT, 
String.valueOf(kafkaConfig.getTimeout()));

[26/32] kylin git commit: KYLIN-2147 Move the creation of HTable after cube be built

[shaofengshi]

KYLIN-2147 Move the creation of HTable after cube be built

Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b6f608f7
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b6f608f7
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b6f608f7

Branch: refs/heads/v1.6.0-rc1-hbase1.x
Commit: b6f608f7e145b32f2c9c50bb2b719b40c1c2eb8b
Parents: 1d8e8e5
Author: shaofengshi 
Authored: Wed Nov 2 13:39:26 2016 +0800
Committer: shaofengshi 
Committed: Wed Nov 2 13:42:14 2016 +0800

--
 .../apache/kylin/storage/hbase/steps/HBaseMROutput2Transition.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/b6f608f7/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2Transition.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2Transition.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2Transition.java
index c4df354..a063efe 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2Transition.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2Transition.java
@@ -49,11 +49,11 @@ public class HBaseMROutput2Transition implements IMROutput2 
{
 
 @Override
 public void addStepPhase2_BuildDictionary(DefaultChainedExecutable 
jobFlow) {
-
jobFlow.addTask(steps.createCreateHTableStepWithStats(jobFlow.getId()));
 }
 
 @Override
 public void addStepPhase3_BuildCube(DefaultChainedExecutable 
jobFlow) {
+
jobFlow.addTask(steps.createCreateHTableStepWithStats(jobFlow.getId()));
 
jobFlow.addTask(steps.createConvertCuboidToHfileStep(jobFlow.getId()));
 jobFlow.addTask(steps.createBulkLoadStep(jobFlow.getId()));
 }

[11/32] kylin git commit: Revert "KYLIN-2137 add check for 'kylin.job.mr.lib.dir'"

[shaofengshi]

Revert "KYLIN-2137 add check for 'kylin.job.mr.lib.dir'"

This reverts commit a37ac46ace9fceda9780121d97348d6eb620f008.


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/11c2c691
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/11c2c691
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/11c2c691

Branch: refs/heads/v1.6.0-rc1-hbase1.x
Commit: 11c2c6919e7d2362c9aef5dca823752651615d57
Parents: a37ac46
Author: Li Yang 
Authored: Sat Oct 29 06:53:23 2016 +0800
Committer: Li Yang 
Committed: Sat Oct 29 06:53:23 2016 +0800

--
 .../apache/kylin/engine/mr/common/AbstractHadoopJob.java| 9 +
 1 file changed, 1 insertion(+), 8 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/11c2c691/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
index a97a8c7..417d63d 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
@@ -245,14 +245,7 @@ public abstract class AbstractHadoopJob extends Configured 
implements Tool {
 
 // for KylinJobMRLibDir
 String mrLibDir = kylinConf.getKylinJobMRLibDir();
-if (!StringUtils.isBlank(mrLibDir)) {
-File dirFileMRLIB = new File(mrLibDir);
-if (dirFileMRLIB.exists()) {
-StringUtil.appendWithSeparator(kylinDependency, mrLibDir);
-} else {
-logger.info("The directory '" + mrLibDir + "' for 
'kylin.job.mr.lib.dir' does not exist!!!");
-}
-}
+StringUtil.appendWithSeparator(kylinDependency, mrLibDir);
 
 setJobTmpJarsAndFiles(job, kylinDependency.toString());
 

[14/32] kylin git commit: KYLIN-2116 Remove field delimiter in hive flatten table ddl

[shaofengshi]

KYLIN-2116 Remove field delimiter in hive flatten table ddl


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/82e99078
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/82e99078
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/82e99078

Branch: refs/heads/v1.6.0-rc1-hbase1.x
Commit: 82e99078c08505b426d34ad6395a14b3d6244b07
Parents: eabbd4e
Author: lidongsjtu 
Authored: Wed Oct 26 22:49:19 2016 +0800
Committer: lidongsjtu 
Committed: Mon Oct 31 13:46:36 2016 +0800

--
 core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java | 2 --
 1 file changed, 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/82e99078/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
--
diff --git a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java 
b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
index 79cf924..c5f38e7 100644
--- a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
+++ b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
@@ -92,8 +92,6 @@ public class JoinedFlatTable {
 ddl.append(colName(col.getCanonicalName()) + " " + 
getHiveDataType(col.getDatatype()) + "\n");
 }
 ddl.append(")" + "\n");
-
-ddl.append("ROW FORMAT DELIMITED FIELDS TERMINATED BY '\\177'" + "\n");
 ddl.append("STORED AS SEQUENCEFILE" + "\n");
 ddl.append("LOCATION '" + getTableDir(flatDesc, storageDfsDir) + 
"';").append("\n");
 // ddl.append("TBLPROPERTIES ('serialization.null.format'='N')" +

[10/32] kylin git commit: KYLIN-2137 add check for 'kylin.job.mr.lib.dir'

[shaofengshi]

KYLIN-2137 add check for 'kylin.job.mr.lib.dir'


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a37ac46a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a37ac46a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a37ac46a

Branch: refs/heads/v1.6.0-rc1-hbase1.x
Commit: a37ac46ace9fceda9780121d97348d6eb620f008
Parents: e419681
Author: shaofengshi 
Authored: Fri Oct 28 22:17:04 2016 +0800
Committer: shaofengshi 
Committed: Fri Oct 28 22:17:04 2016 +0800

--
 .../apache/kylin/engine/mr/common/AbstractHadoopJob.java| 9 -
 1 file changed, 8 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/a37ac46a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
index 417d63d..a97a8c7 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
@@ -245,7 +245,14 @@ public abstract class AbstractHadoopJob extends Configured 
implements Tool {
 
 // for KylinJobMRLibDir
 String mrLibDir = kylinConf.getKylinJobMRLibDir();
-StringUtil.appendWithSeparator(kylinDependency, mrLibDir);
+if (!StringUtils.isBlank(mrLibDir)) {
+File dirFileMRLIB = new File(mrLibDir);
+if (dirFileMRLIB.exists()) {
+StringUtil.appendWithSeparator(kylinDependency, mrLibDir);
+} else {
+logger.info("The directory '" + mrLibDir + "' for 
'kylin.job.mr.lib.dir' does not exist!!!");
+}
+}
 
 setJobTmpJarsAndFiles(job, kylinDependency.toString());
 

[24/32] kylin git commit: KYLIN-2114 remove builder dictionary config to cubeConfig

[shaofengshi]

KYLIN-2114 remove builder dictionary config to cubeConfig


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/eaaaf2f7
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/eaaaf2f7
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/eaaaf2f7

Branch: refs/heads/v1.6.0-rc1-hbase1.x
Commit: eaaaf2f7bbe3ec617cd827eeeac31cdad8e8950e
Parents: c780e43
Author: Jason 
Authored: Tue Nov 1 19:15:48 2016 +0800
Committer: Jason 
Committed: Tue Nov 1 19:16:21 2016 +0800

--
 webapp/app/js/model/cubeConfig.js   | 5 -
 webapp/app/partials/cubeDesigner/advanced_settings.html | 2 +-
 2 files changed, 5 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/eaaaf2f7/webapp/app/js/model/cubeConfig.js
--
diff --git a/webapp/app/js/model/cubeConfig.js 
b/webapp/app/js/model/cubeConfig.js
index a77ab4f..87413dc 100644
--- a/webapp/app/js/model/cubeConfig.js
+++ b/webapp/app/js/model/cubeConfig.js
@@ -104,5 +104,8 @@ KylinApp.constant('cubeConfig', {
   rowKeyShardOptions:[
 true,false
   ],
-  statusNeedNofity:['ERROR', 'DISCARDED', 'SUCCEED']
+  statusNeedNofity:['ERROR', 'DISCARDED', 'SUCCEED'],
+  buildDictionaries:[
+{name:"Global Dictionary", 
value:"org.apache.kylin.dict.GlobalDictionaryBuilder"}
+  ]
 });

http://git-wip-us.apache.org/repos/asf/kylin/blob/eaaaf2f7/webapp/app/partials/cubeDesigner/advanced_settings.html
--
diff --git a/webapp/app/partials/cubeDesigner/advanced_settings.html 
b/webapp/app/partials/cubeDesigner/advanced_settings.html
index a1e3c80..0a9dc48 100755
--- a/webapp/app/partials/cubeDesigner/advanced_settings.html
+++ b/webapp/app/partials/cubeDesigner/advanced_settings.html
@@ -370,8 +370,8 @@
  

- Global Dictionary
  -- Select a builder class--

  

[13/32] kylin git commit: remove fitler hive dependency logic in AbstractHadoopJob

[shaofengshi]

remove fitler hive dependency logic in AbstractHadoopJob


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/eabbd4e1
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/eabbd4e1
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/eabbd4e1

Branch: refs/heads/v1.6.0-rc1-hbase1.x
Commit: eabbd4e16869150625af43b3cefa5e9076378f48
Parents: 6bdd4f3
Author: Hongbin Ma 
Authored: Thu Oct 13 21:35:49 2016 +0800
Committer: Hongbin Ma 
Committed: Mon Oct 31 12:46:08 2016 +0800

--
 .../java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/eabbd4e1/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
index 417d63d..72f4437 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
@@ -200,7 +200,8 @@ public abstract class AbstractHadoopJob extends Configured 
implements Tool {
 kylinHiveDependency = kylinHiveDependency.replace(":", ",");
 
 logger.info("Hive Dependencies Before Filtered: " + 
kylinHiveDependency);
-String filteredHive = 
filterKylinHiveDependency(kylinHiveDependency);
+//String filteredHive = 
filterKylinHiveDependency(kylinHiveDependency);
+String filteredHive = kylinHiveDependency;
 logger.info("Hive Dependencies After Filtered: " + filteredHive);
 
 StringUtil.appendWithSeparator(kylinDependency, filteredHive);

[32/32] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[shaofengshi]

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3940e80f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3940e80f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3940e80f

Branch: refs/heads/v1.6.0-rc1-hbase1.x
Commit: 3940e80f9565076bfbff3a0b4f7f847e747160b2
Parents: 7a37930
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: shaofengshi 
Committed: Wed Nov 2 16:33:28 2016 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 13 ++-
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 35 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  6 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 23 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 36 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java|  2 +-
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 48 files changed, 392 insertions(+), 448 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/3940e80f/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-etc.  Retry interval is a rough function based on 
hbase.client.pause.  At
-first we retry at this interval but then with backoff, we pretty 
quickly reach
-retrying every ten seconds.  See HConstants#RETRY_BACKOFF for how 
the backup
-ramps up.  

[04/32] kylin git commit: KYLIN-2125 add BeelineHiveClient

[shaofengshi]

KYLIN-2125 add BeelineHiveClient


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9ed775c3
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9ed775c3
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9ed775c3

Branch: refs/heads/v1.6.0-rc1-hbase1.x
Commit: 9ed775c35426fea47c9c63a37677d73cddb70e9c
Parents: e094b76
Author: Hongbin Ma 
Authored: Thu Oct 13 15:31:11 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Oct 27 08:34:11 2016 +0800

--
 .../kylin/rest/controller/TableController.java  |   9 +-
 source-hive/pom.xml |   5 +
 .../kylin/source/hive/BeelineHiveClient.java| 214 +++
 .../source/hive/BeelineOptionsProcessor.java|  47 
 .../apache/kylin/source/hive/CLIHiveClient.java | 169 +++
 .../apache/kylin/source/hive/HiveClient.java| 170 ---
 .../kylin/source/hive/HiveClientFactory.java|  33 +++
 .../source/hive/HiveSourceTableLoader.java  |  89 
 .../org/apache/kylin/source/hive/HiveTable.java |  16 +-
 .../apache/kylin/source/hive/HiveTableMeta.java |  71 ++
 .../kylin/source/hive/HiveTableMetaBuilder.java | 102 +
 .../apache/kylin/source/hive/HqlExecutable.java | 107 --
 .../apache/kylin/source/hive/IHiveClient.java   |  36 
 .../hive/BeelineOptionsProcessorTest.java   |  38 
 14 files changed, 769 insertions(+), 337 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/9ed775c3/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
 
b/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
index d9050c1..47ff3fe 100644
--- 
a/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
+++ 
b/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
@@ -47,7 +47,8 @@ import org.apache.kylin.rest.service.KafkaConfigService;
 import org.apache.kylin.rest.service.ModelService;
 import org.apache.kylin.rest.service.ProjectService;
 import org.apache.kylin.rest.service.StreamingService;
-import org.apache.kylin.source.hive.HiveClient;
+import org.apache.kylin.source.hive.HiveClientFactory;
+import org.apache.kylin.source.hive.IHiveClient;
 import org.apache.kylin.source.kafka.config.KafkaConfig;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -189,7 +190,7 @@ public class TableController extends BasicController {
 String[] dbTableName = HadoopUtil.parseHiveTableName(tableName);
 tableName = dbTableName[0] + "." + dbTableName[1];
 TableDesc desc = 
cubeMgmtService.getMetadataManager().getTableDesc(tableName);
-if(desc == null)
+if (desc == null)
 return false;
 tableType = desc.getSourceType();
 
@@ -315,7 +316,7 @@ public class TableController extends BasicController {
 @RequestMapping(value = "/hive", method = { RequestMethod.GET })
 @ResponseBody
 private static List showHiveDatabases() throws IOException {
-HiveClient hiveClient = new HiveClient();
+IHiveClient hiveClient = HiveClientFactory.getHiveClient();
 List results = null;
 
 try {
@@ -336,7 +337,7 @@ public class TableController extends BasicController {
 @RequestMapping(value = "/hive/{database}", method = { RequestMethod.GET })
 @ResponseBody
 private static List showHiveTables(@PathVariable String database) 
throws IOException {
-HiveClient hiveClient = new HiveClient();
+IHiveClient hiveClient = HiveClientFactory.getHiveClient();
 List results = null;
 
 try {

http://git-wip-us.apache.org/repos/asf/kylin/blob/9ed775c3/source-hive/pom.xml
--
diff --git a/source-hive/pom.xml b/source-hive/pom.xml
index 08019d0..16cb3b4 100644
--- a/source-hive/pom.xml
+++ b/source-hive/pom.xml
@@ -65,6 +65,11 @@
 provided
 
 
+org.apache.hive
+hive-jdbc
+provided
+
+
 org.apache.mrunit
 mrunit
 hadoop2

http://git-wip-us.apache.org/repos/asf/kylin/blob/9ed775c3/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
--
diff --git 
a/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java 
b/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
new file mode 100644
index 000..0fbc39b
--- /dev/null
+++ 

[23/32] kylin git commit: KYLIN-1904-WEB-Global-Dictionary bug fix

[shaofengshi]

KYLIN-1904-WEB-Global-Dictionary bug fix

Signed-off-by: Jason 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c780e43e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c780e43e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c780e43e

Branch: refs/heads/v1.6.0-rc1-hbase1.x
Commit: c780e43ea2fa1b35e8d2338dd4a8e37ba3e427c4
Parents: 3f1a495
Author: kangkaisen 
Authored: Thu Oct 20 13:31:43 2016 +0800
Committer: Jason 
Committed: Tue Nov 1 17:30:33 2016 +0800

--
 webapp/app/js/controllers/cubeEdit.js   | 5 +++--
 webapp/app/partials/cubeDesigner/advanced_settings.html | 7 ++-
 2 files changed, 9 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/c780e43e/webapp/app/js/controllers/cubeEdit.js
--
diff --git a/webapp/app/js/controllers/cubeEdit.js 
b/webapp/app/js/controllers/cubeEdit.js
index 06d3120..09dae15 100755
--- a/webapp/app/js/controllers/cubeEdit.js
+++ b/webapp/app/js/controllers/cubeEdit.js
@@ -206,8 +206,9 @@ KylinApp.controller('CubeEditCtrl', function ($scope, $q, 
$routeParams, $locatio
   }
 
 });
-angular.forEach($scope.cubeMetaFrame.measure,function(measure){
- if(measure.function.parameter.type==column){
+
+angular.forEach($scope.cubeMetaFrame.measures,function(measure){
+ if(measure.function.parameter.type == "column"){
me_columns.push(measure.function.parameter.value);
  }
 });

http://git-wip-us.apache.org/repos/asf/kylin/blob/c780e43e/webapp/app/partials/cubeDesigner/advanced_settings.html
--
diff --git a/webapp/app/partials/cubeDesigner/advanced_settings.html 
b/webapp/app/partials/cubeDesigner/advanced_settings.html
index 34fd492..a1e3c80 100755
--- a/webapp/app/partials/cubeDesigner/advanced_settings.html
+++ b/webapp/app/partials/cubeDesigner/advanced_settings.html
@@ -368,7 +368,12 @@

  Builder Class
  
-   
+   
+ Global Dictionary
+ -- Select a builder class--
+   
  

  

[05/32] kylin git commit: KYLIN-2134 Kylin will treat empty string as NULL by mistake

[shaofengshi]

KYLIN-2134 Kylin will treat empty string as NULL by mistake


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1af3d9ee
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1af3d9ee
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1af3d9ee

Branch: refs/heads/v1.6.0-rc1-hbase1.x
Commit: 1af3d9ee31f3edd64eea336c65b7619b078efdbf
Parents: cd2a06a
Author: Hongbin Ma 
Authored: Thu Oct 27 15:22:54 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Oct 27 15:25:48 2016 +0800

--
 .../src/main/java/org/apache/kylin/metadata/tuple/Tuple.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/1af3d9ee/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/Tuple.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/Tuple.java 
b/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/Tuple.java
index aaf9aa9..9074403 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/Tuple.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/Tuple.java
@@ -178,7 +178,7 @@ public class Tuple implements ITuple {
 if (strValue == null)
 return null;
 
-if ((strValue.equals("") || strValue.equals("\\N")) && 
!dataTypeName.equals("string"))
+if ((strValue.equals("") || strValue.equals("\\N")) && 
!dataTypeName.equals("string") && !dataTypeName.startsWith("varchar"))
 return null;
 
 // TODO use data type enum instead of string comparison

[30/32] kylin git commit: view cube bug

[shaofengshi]

view cube bug

Signed-off-by: shaofengshi 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7a379305
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7a379305
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7a379305

Branch: refs/heads/v1.6.0-rc1-hbase1.x
Commit: 7a3793059b5e790ba0de667b7feaabae7a468119
Parents: 7d13664
Author: luguosheng <550175...@qq.com>
Authored: Wed Nov 2 16:17:15 2016 +0800
Committer: shaofengshi 
Committed: Wed Nov 2 16:26:58 2016 +0800

--
 webapp/app/js/controllers/cubeEdit.js | 8 
 1 file changed, 4 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/7a379305/webapp/app/js/controllers/cubeEdit.js
--
diff --git a/webapp/app/js/controllers/cubeEdit.js 
b/webapp/app/js/controllers/cubeEdit.js
index 09dae15..acd4d32 100755
--- a/webapp/app/js/controllers/cubeEdit.js
+++ b/webapp/app/js/controllers/cubeEdit.js
@@ -293,11 +293,11 @@ KylinApp.controller('CubeEditCtrl', function ($scope, $q, 
$routeParams, $locatio
 });
 
 var queryParam = {
-  cube_name: $routeParams.cubeName
+  cubeId: $routeParams.cubeName
 };
-CubeService.list(queryParam, {},function(instance){
-  if (instance.length > 0) {
-$scope.instance = instance[0];
+CubeService.getCube(queryParam, {},function(instance){
+  if (instance) {
+$scope.instance = instance;
 $scope.state.cubeInstance =angular.toJson($scope.instance,true);
 
   } else {

[08/32] kylin git commit: KYLIN-2105 remove QueryIdGenerator and QueryIdGeneratorTest, simply use UUID as QueryId

[shaofengshi]

KYLIN-2105 remove QueryIdGenerator and QueryIdGeneratorTest, simply use UUID as 
QueryId


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/46a894db
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/46a894db
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/46a894db

Branch: refs/heads/v1.6.0-rc1-hbase1.x
Commit: 46a894db60e7538468d6e7459ea760bca286249e
Parents: 5d4d639
Author: gaodayue 
Authored: Thu Oct 27 12:14:49 2016 +0800
Committer: gaodayue 
Committed: Fri Oct 28 13:32:38 2016 +0800

--
 .../apache/kylin/rest/service/QueryService.java |   8 +-
 .../kylin/rest/util/QueryIdGenerator.java   |  47 
 .../kylin/rest/util/QueryIdGeneratorTest.java   | 108 ---
 .../coprocessor/endpoint/CubeVisitService.java  |   2 +-
 4 files changed, 4 insertions(+), 161 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/46a894db/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java 
b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
index 94f2dd5..81af044 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -39,6 +39,7 @@ import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.UUID;
 
 import javax.annotation.PostConstruct;
 import javax.sql.DataSource;
@@ -72,7 +73,6 @@ import org.apache.kylin.rest.model.TableMeta;
 import org.apache.kylin.rest.request.PrepareSqlRequest;
 import org.apache.kylin.rest.request.SQLRequest;
 import org.apache.kylin.rest.response.SQLResponse;
-import org.apache.kylin.rest.util.QueryIdGenerator;
 import org.apache.kylin.rest.util.QueryUtil;
 import org.apache.kylin.rest.util.Serializer;
 import org.apache.kylin.rest.util.TableauInterceptor;
@@ -117,8 +117,6 @@ public class QueryService extends BasicService {
 private final String hbaseUrl;
 private final String userTableName;
 
-private QueryIdGenerator queryIdGenerator = new QueryIdGenerator();
-
 @Autowired
 private CacheManager cacheManager;
 
@@ -325,7 +323,7 @@ public class QueryService extends BasicService {
 throw new InternalErrorException("Query is not allowed in " + 
serverMode + " mode.");
 }
 
-final String queryId = 
queryIdGenerator.nextId(sqlRequest.getProject());
+final String queryId = UUID.randomUUID().toString();
 
 Map toggles = new HashMap<>();
 toggles.put(BackdoorToggles.KEY_QUERY_ID, queryId);
@@ -334,7 +332,7 @@ public class QueryService extends BasicService {
 }
 BackdoorToggles.setToggles(toggles);
 
-try (SetThreadName ignored = new SetThreadName("Query-%s", queryId)) {
+try (SetThreadName ignored = new SetThreadName("Query %s", queryId)) {
 String sql = sqlRequest.getSql();
 String project = sqlRequest.getProject();
 logger.info("Using project: " + project);

http://git-wip-us.apache.org/repos/asf/kylin/blob/46a894db/server-base/src/main/java/org/apache/kylin/rest/util/QueryIdGenerator.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/util/QueryIdGenerator.java 
b/server-base/src/main/java/org/apache/kylin/rest/util/QueryIdGenerator.java
deleted file mode 100644
index 2dd19c2..000
--- a/server-base/src/main/java/org/apache/kylin/rest/util/QueryIdGenerator.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.rest.util;
-
-import org.apache.commons.lang3.time.FastDateFormat;
-
-import javax.annotation.Nonnull;
-import javax.annotation.concurrent.ThreadSafe;
-import 

[27/32] kylin git commit: KYLIN-2122 Move the partition offset calculation before submitting job

[shaofengshi]

KYLIN-2122 Move the partition offset calculation before submitting job


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2beccbf9
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2beccbf9
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2beccbf9

Branch: refs/heads/v1.6.0-rc1-hbase1.x
Commit: 2beccbf94fd8e43c072959aeec98cc05d8552788
Parents: b6f608f
Author: shaofengshi 
Authored: Tue Oct 25 11:54:22 2016 +0800
Committer: shaofengshi 
Committed: Wed Nov 2 13:47:34 2016 +0800

--
 .../org/apache/kylin/cube/CubeInstance.java |   9 ++
 .../java/org/apache/kylin/cube/CubeManager.java | 103 +-
 .../org/apache/kylin/cube/CubeManagerTest.java  |   2 +-
 .../org/apache/kylin/cube/CubeSegmentsTest.java |   2 +-
 .../kylin/job/constant/ExecutableConstants.java |   1 +
 .../java/org/apache/kylin/source/ISource.java   |  13 ++-
 .../org/apache/kylin/source/SourceFactory.java  |   2 +-
 .../apache/kylin/source/SourcePartition.java| 103 ++
 .../kylin/engine/mr/JobBuilderSupport.java  |   2 +
 .../mr/steps/UpdateCubeInfoAfterBuildStep.java  |  67 
 .../kylin/provision/BuildCubeWithEngine.java|   8 +-
 .../kylin/provision/BuildCubeWithStream.java|  17 ++-
 .../apache/kylin/rest/service/JobService.java   |  16 +--
 .../apache/kylin/source/hive/HiveSource.java|  32 +-
 .../apache/kylin/source/kafka/KafkaMRInput.java |  90 ++--
 .../apache/kylin/source/kafka/KafkaSource.java  | 105 +-
 .../kylin/source/kafka/config/KafkaConfig.java  |   3 +
 .../source/kafka/hadoop/KafkaInputFormat.java   |  41 ---
 .../kylin/source/kafka/job/SeekOffsetStep.java  | 106 +--
 .../source/kafka/job/UpdateTimeRangeStep.java   |  78 +-
 .../kylin/source/kafka/util/KafkaClient.java|  24 -
 21 files changed, 483 insertions(+), 341 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/2beccbf9/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
index 720690d..7222457 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
@@ -439,6 +439,15 @@ public class CubeInstance extends RootPersistentEntity 
implements IRealization,
 return this.getDescriptor().getAutoMergeTimeRanges() != null && 
this.getDescriptor().getAutoMergeTimeRanges().length > 0;
 }
 
+public CubeSegment getLastSegment() {
+List existing = getSegments();
+if (existing.isEmpty()) {
+return null;
+} else {
+return existing.get(existing.size() - 1);
+}
+}
+
 @Override
 public int getSourceType() {
 return getFactTableDesc().getSourceType();

http://git-wip-us.apache.org/repos/asf/kylin/blob/2beccbf9/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index a53849e..16b468f 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -34,8 +34,6 @@ import java.util.Random;
 import java.util.UUID;
 import java.util.concurrent.ConcurrentHashMap;
 
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Maps;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.KylinConfigExt;
@@ -68,9 +66,11 @@ import 
org.apache.kylin.metadata.realization.RealizationStatusEnum;
 import org.apache.kylin.metadata.realization.RealizationType;
 import org.apache.kylin.source.ReadableTable;
 import org.apache.kylin.source.SourceFactory;
+import org.apache.kylin.source.SourcePartition;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.google.common.base.Preconditions;
 import com.google.common.collect.HashMultimap;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Multimap;
@@ -434,52 +434,20 @@ public class CubeManager implements IRealizationProvider {
 
 // append a full build segment
 public CubeSegment appendSegment(CubeInstance cube) throws IOException {
-return appendSegment(cube, 0, 0, 0, 0, null, null);
+return appendSegment(cube, 0, Long.MAX_VALUE, 0, 0, null, null);
 }
 
 public CubeSegment 

[06/32] kylin git commit: KYLIN-2134 dont reuse Object[] returned to calcite

[shaofengshi]

KYLIN-2134 dont reuse Object[] returned to calcite


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8199a90c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8199a90c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8199a90c

Branch: refs/heads/v1.6.0-rc1-hbase1.x
Commit: 8199a90c57df12e8ad4a19737ffa9aa17a69f3eb
Parents: 1af3d9e
Author: Li Yang 
Authored: Thu Oct 27 17:09:22 2016 +0800
Committer: Hongbin Ma 
Committed: Fri Oct 28 09:51:51 2016 +0800

--
 .../kylin/storage/gtrecord/SegmentCubeTupleIterator.java | 4 
 .../org/apache/kylin/query/enumerator/OLAPEnumerator.java| 8 
 2 files changed, 4 insertions(+), 8 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/8199a90c/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SegmentCubeTupleIterator.java
--
diff --git 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SegmentCubeTupleIterator.java
 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SegmentCubeTupleIterator.java
index 61267ae..00ba247 100644
--- 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SegmentCubeTupleIterator.java
+++ 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SegmentCubeTupleIterator.java
@@ -37,8 +37,6 @@ import org.apache.kylin.storage.StorageContext;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.google.common.base.Preconditions;
-
 public class SegmentCubeTupleIterator implements ITupleIterator {
 
 private static final Logger logger = 
LoggerFactory.getLogger(SegmentCubeTupleIterator.class);
@@ -98,8 +96,6 @@ public class SegmentCubeTupleIterator implements 
ITupleIterator {
 }
 GTRecord curRecord = gtItr.next();
 
-Preconditions.checkNotNull(cubeTupleConverter);
-
 // translate into tuple
 advMeasureFillers = cubeTupleConverter.translateResult(curRecord, 
tuple);
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/8199a90c/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
--
diff --git 
a/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java 
b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
index dafbb5f..c7b3c71 100644
--- a/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
+++ b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
@@ -18,6 +18,7 @@
 
 package org.apache.kylin.query.enumerator;
 
+import java.util.Arrays;
 import java.util.Map;
 import java.util.Properties;
 
@@ -77,10 +78,9 @@ public class OLAPEnumerator implements Enumerator {
 }
 
 private Object[] convertCurrentRow(ITuple tuple) {
-// make sure the tuple layout is correct
-//assert 
tuple.getAllFields().equals(olapContext.returnTupleInfo.getAllFields());
-
-current = tuple.getAllValues();
+// give calcite a new array every time, see details in KYLIN-2134
+Object[] values = tuple.getAllValues();
+current = Arrays.copyOf(values, values.length);
 return current;
 }
 

[18/32] kylin git commit: KYLIN-1321 Add derived checkbox for lookup table columns on Auto Generate Dimensions panel, style update

[shaofengshi]

KYLIN-1321 Add derived checkbox for lookup table columns on Auto Generate 
Dimensions panel, style update


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/21ecfce6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/21ecfce6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/21ecfce6

Branch: refs/heads/v1.6.0-rc1-hbase1.x
Commit: 21ecfce6d0593c891c4cf7a6360c5ee29b01710c
Parents: d01a1f7
Author: Jason 
Authored: Mon Oct 31 17:08:14 2016 +0800
Committer: Jason 
Committed: Mon Oct 31 17:10:07 2016 +0800

--
 webapp/app/partials/cubeDesigner/dimensions.html | 10 +-
 1 file changed, 5 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/21ecfce6/webapp/app/partials/cubeDesigner/dimensions.html
--
diff --git a/webapp/app/partials/cubeDesigner/dimensions.html 
b/webapp/app/partials/cubeDesigner/dimensions.html
index 62ca5c9..9539250 100644
--- a/webapp/app/partials/cubeDesigner/dimensions.html
+++ b/webapp/app/partials/cubeDesigner/dimensions.html
@@ -240,10 +240,10 @@
 {{table}}[FactTable]
 
   
-Select All
+Select All
 Name
 Columns
-
+
   
   
 
@@ -266,10 +266,10 @@
 {{table}}[LookupTable]
 
   
-Select All
+Select All
 Name
 Columns
-
+
   
   
 
@@ -299,8 +299,8 @@
 
 
 
-OK
 Cancel
+OK
 
 
 

[09/32] kylin git commit: KYLIN-2137 Failed to run mr job when user put a kafka jar in hive's lib folder

[shaofengshi]

KYLIN-2137 Failed to run mr job when user put a kafka jar in hive's lib folder

Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e419681e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e419681e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e419681e

Branch: refs/heads/v1.6.0-rc1-hbase1.x
Commit: e419681e8c9c6135af40dcdc2b9b3b367b3e2456
Parents: 46a894d
Author: shaofengshi 
Authored: Fri Oct 28 18:15:33 2016 +0800
Committer: shaofengshi 
Committed: Fri Oct 28 22:11:33 2016 +0800

--
 .../apache/kylin/common/util/StringUtil.java| 13 +
 .../engine/mr/common/AbstractHadoopJob.java | 29 ++--
 2 files changed, 22 insertions(+), 20 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/e419681e/core-common/src/main/java/org/apache/kylin/common/util/StringUtil.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/util/StringUtil.java 
b/core-common/src/main/java/org/apache/kylin/common/util/StringUtil.java
index 15872aa..b336e4b 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/StringUtil.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/StringUtil.java
@@ -124,4 +124,17 @@ public class StringUtil {
 return result;
 }
 
+public static void appendWithSeparator(StringBuilder src, String append) {
+if (src == null) {
+throw new IllegalArgumentException();
+}
+if (src.length() > 0 && src.toString().endsWith(",") == false) {
+src.append(",");
+}
+
+if (StringUtils.isBlank(append) == false) {
+src.append(append);
+}
+}
+
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/e419681e/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
index a5b2d2e..417d63d 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
@@ -61,6 +61,7 @@ import org.apache.kylin.common.persistence.ResourceStore;
 import org.apache.kylin.common.util.CliCommandExecutor;
 import org.apache.kylin.common.util.OptionsHelper;
 import org.apache.kylin.common.util.StringSplitter;
+import org.apache.kylin.common.util.StringUtil;
 import org.apache.kylin.cube.CubeInstance;
 import org.apache.kylin.cube.CubeSegment;
 import org.apache.kylin.engine.mr.HadoopUtil;
@@ -202,24 +203,23 @@ public abstract class AbstractHadoopJob extends 
Configured implements Tool {
 String filteredHive = 
filterKylinHiveDependency(kylinHiveDependency);
 logger.info("Hive Dependencies After Filtered: " + filteredHive);
 
-if (kylinDependency.length() > 0)
-kylinDependency.append(",");
-kylinDependency.append(filteredHive);
+StringUtil.appendWithSeparator(kylinDependency, filteredHive);
 } else {
 
 logger.info("No hive dependency jars set in the environment, will 
find them from jvm:");
 
 try {
 String hiveExecJarPath = 
ClassUtil.findContainingJar(Class.forName("org.apache.hadoop.hive.ql.Driver"));
-kylinDependency.append(hiveExecJarPath).append(",");
+
+StringUtil.appendWithSeparator(kylinDependency, 
hiveExecJarPath);
 logger.info("hive-exec jar file: " + hiveExecJarPath);
 
 String hiveHCatJarPath = 
ClassUtil.findContainingJar(Class.forName("org.apache.hive.hcatalog.mapreduce.HCatInputFormat"));
-kylinDependency.append(hiveHCatJarPath).append(",");
+StringUtil.appendWithSeparator(kylinDependency, 
hiveHCatJarPath);
 logger.info("hive-catalog jar file: " + hiveHCatJarPath);
 
 String hiveMetaStoreJarPath = 
ClassUtil.findContainingJar(Class.forName("org.apache.hadoop.hive.metastore.api.Table"));
-kylinDependency.append(hiveMetaStoreJarPath).append(",");
+StringUtil.appendWithSeparator(kylinDependency, 
hiveMetaStoreJarPath);
 logger.info("hive-metastore jar file: " + 
hiveMetaStoreJarPath);
 } catch (ClassNotFoundException e) {
 logger.error("Cannot found hive dependency jars: " + e);
@@ -229,19 +229,13 @@ public abstract class AbstractHadoopJob extends 
Configured implements Tool {
 // for kafka dependencies
  

[28/32] kylin git commit: KYLIN-2122 fix class visibility error

[shaofengshi]

KYLIN-2122 fix class visibility error


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4a689b2d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4a689b2d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4a689b2d

Branch: refs/heads/v1.6.0-rc1-hbase1.x
Commit: 4a689b2db039aaa0dcc9cb3ce50192f11961b8a4
Parents: 2beccbf
Author: shaofengshi 
Authored: Wed Nov 2 13:58:29 2016 +0800
Committer: shaofengshi 
Committed: Wed Nov 2 13:58:29 2016 +0800

--
 .../src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/4a689b2d/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java
--
diff --git 
a/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java 
b/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java
index cdd7272..99ebb6a 100644
--- a/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java
+++ b/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java
@@ -208,7 +208,7 @@ public class KafkaMRInput implements IMRInput {
 }
 }
 
-static class GarbageCollectionStep extends AbstractExecutable {
+public static class GarbageCollectionStep extends AbstractExecutable {
 private static final Logger logger = 
LoggerFactory.getLogger(GarbageCollectionStep.class);
 
 @Override

[17/32] kylin git commit: KYLIN Add derived checkbox for lookup table columns on Auto Generate Dimensions panel

[shaofengshi]

KYLIN Add derived checkbox for lookup table columns on Auto Generate Dimensions 
panel

Signed-off-by: Jason 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d01a1f74
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d01a1f74
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d01a1f74

Branch: refs/heads/v1.6.0-rc1-hbase1.x
Commit: d01a1f7433a2529c5075f09786a50a86d5ea4653
Parents: 5218c9f
Author: chenzhx <346839...@qq.com>
Authored: Fri Oct 28 15:45:22 2016 +0800
Committer: Jason 
Committed: Mon Oct 31 17:09:58 2016 +0800

--
 webapp/app/js/controllers/cubeDimensions.js | 110 +++
 .../app/partials/cubeDesigner/dimensions.html   |  26 ++---
 2 files changed, 100 insertions(+), 36 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/d01a1f74/webapp/app/js/controllers/cubeDimensions.js
--
diff --git a/webapp/app/js/controllers/cubeDimensions.js 
b/webapp/app/js/controllers/cubeDimensions.js
index ba71c96..7cb850b 100644
--- a/webapp/app/js/controllers/cubeDimensions.js
+++ b/webapp/app/js/controllers/cubeDimensions.js
@@ -74,11 +74,12 @@ KylinApp.controller('CubeDimensionsCtrl', function ($scope, 
$modal,MetaModel,cub
 cols[i].isLookup = false;
 
 // Default not selected and not disabled.
-factSelectAvailable[cols[i].name] = {name:cols[i].name ,selected: 
false, disabled: false};
+factSelectAvailable[cols[i].name] = {name:cols[i].name ,selected: 
false};
 
 }
 
 $scope.availableColumns[factTable] = cols;
+factSelectAvailable.all=false;
 $scope.selectedColumns[factTable] = factSelectAvailable;
 $scope.availableTables.push(factTable);
 
@@ -96,10 +97,11 @@ KylinApp.controller('CubeDimensionsCtrl', function ($scope, 
$modal,MetaModel,cub
 cols2[k].isLookup = true;
 
 // Default not selected and not disabled.
-lookupSelectAvailable[cols2[k].name] = 
{name:cols2[k].name,selected: false, disabled: false};
+lookupSelectAvailable[cols2[k].name] = 
{name:cols2[k].name,selected: false};
 }
 
 $scope.availableColumns[lookups[j].table] = cols2;
+lookupSelectAvailable.all=false;
 $scope.selectedColumns[lookups[j].table] = lookupSelectAvailable;
 if($scope.availableTables.indexOf(lookups[j].table)==-1){
 $scope.availableTables.push(lookups[j].table);
@@ -111,14 +113,22 @@ KylinApp.controller('CubeDimensionsCtrl', function 
($scope, $modal,MetaModel,cub
 $scope.initColumnStatus = function () {
 angular.forEach($scope.cubeMetaFrame.dimensions, function (dim) {
 var cols = dimCols(dim);
-
 angular.forEach(cols, function (colName) {
   if(dim.derived){
- $scope.selectedColumns[dim.table][colName] = {name:dim.name, 
selected: true, disabled: true,normal:"false"};
+ $scope.selectedColumns[dim.table][colName] = {name:dim.name, 
selected: true, normal:"false"};
   }else{
- $scope.selectedColumns[dim.table][colName] = {name:dim.name, 
selected: true, disabled: true,normal:"true"};
+ $scope.selectedColumns[dim.table][colName] = {name:dim.name, 
selected: true, normal:"true"};
   }
-   });
+});
+});
+angular.forEach($scope.selectedColumns,function(value,table){
+  var all=true;
+  angular.forEach(value,function(col){
+   if(col.selected==false& col=="object"){
+all=false;
+   }
+ });
+ $scope.selectedColumns[table].all=all;
 });
 };
 
@@ -300,9 +310,9 @@ KylinApp.controller('CubeDimensionsCtrl', function ($scope, 
$modal,MetaModel,cub
 var cols = dimCols(dim);
 angular.forEach(cols, function (colName) {
 if(dim.table==$scope.metaModel.model.fact_table){
-   $scope.selectedColumns[dim.table][colName] = 
{name:colName,selected: false, disabled: false};
+   $scope.selectedColumns[dim.table][colName] = 
{name:colName,selected: false};
 }else{
-   $scope.selectedColumns[dim.table][colName] = 
{name:colName,selected: false, disabled: false};
+   $scope.selectedColumns[dim.table][colName] = 
{name:colName,selected: false};
 }
 });
 };
@@ -395,25 +405,77 @@ KylinApp.controller('CubeDimensionsCtrl', function 
($scope, $modal,MetaModel,cub
 };
 
 $scope.autoChange= function(table,name){
-if($scope.selectedColumns[table][name].selected==false){
- 

[19/32] kylin git commit: AbstractHadoopJob: set hive dependency filter as a config entry

[shaofengshi]

AbstractHadoopJob: set hive dependency filter as a config entry


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a8922e87
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a8922e87
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a8922e87

Branch: refs/heads/v1.6.0-rc1-hbase1.x
Commit: a8922e87b1f483dc010185f4a08aae970898c45d
Parents: 21ecfce
Author: Hongbin Ma 
Authored: Mon Oct 31 17:03:42 2016 +0800
Committer: Hongbin Ma 
Committed: Mon Oct 31 17:11:01 2016 +0800

--
 .../main/java/org/apache/kylin/common/KylinConfigBase.java  | 8 ++--
 .../apache/kylin/engine/mr/common/AbstractHadoopJob.java| 9 +++--
 2 files changed, 9 insertions(+), 8 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/a8922e87/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 26c280b..011155f 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -22,9 +22,9 @@ import java.io.File;
 import java.io.IOException;
 import java.io.Serializable;
 import java.util.Map;
+import java.util.Map.Entry;
 import java.util.Properties;
 import java.util.SortedSet;
-import java.util.Map.Entry;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
@@ -504,7 +504,7 @@ abstract public class KylinConfigBase implements 
Serializable {
 public boolean getBadQueryPersistentEnabled() {
 return 
Boolean.parseBoolean(getOptional("kylin.query.badquery.persistent.enable", 
"true"));
 }
-
+
 public String[] getQueryTransformers() {
 return getOptionalStringArray("kylin.query.transformers", new 
String[0]);
 }
@@ -633,6 +633,10 @@ abstract public class KylinConfigBase implements 
Serializable {
 return 
this.getOptional("kylin.job.hive.database.for.intermediatetable", "default");
 }
 
+public String getHiveDependencyFilterList() {
+return this.getOptional("kylin.job.dependency.filterlist", 
"[^,]*hive-exec[0-9.-]+[^,]*?\\.jar" + "|" + 
"[^,]*hive-metastore[0-9.-]+[^,]*?\\.jar" + "|" + 
"[^,]*hive-hcatalog-core[0-9.-]+[^,]*?\\.jar");
+}
+
 public String getKylinOwner() {
 return this.getOptional("kylin.owner", "");
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/a8922e87/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
index 72f4437..77791ce 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
@@ -96,8 +96,6 @@ public abstract class AbstractHadoopJob extends Configured 
implements Tool {
 
 private static final String MAP_REDUCE_CLASSPATH = 
"mapreduce.application.classpath";
 
-private static final String KYLIN_HIVE_DEPENDENCY_JARS = 
"[^,]*hive-exec[0-9.-]+[^,]*?\\.jar" + "|" + 
"[^,]*hive-metastore[0-9.-]+[^,]*?\\.jar" + "|" + 
"[^,]*hive-hcatalog-core[0-9.-]+[^,]*?\\.jar";
-
 protected static void runJob(Tool job, String[] args) {
 try {
 int exitCode = ToolRunner.run(job, args);
@@ -200,8 +198,7 @@ public abstract class AbstractHadoopJob extends Configured 
implements Tool {
 kylinHiveDependency = kylinHiveDependency.replace(":", ",");
 
 logger.info("Hive Dependencies Before Filtered: " + 
kylinHiveDependency);
-//String filteredHive = 
filterKylinHiveDependency(kylinHiveDependency);
-String filteredHive = kylinHiveDependency;
+String filteredHive = 
filterKylinHiveDependency(kylinHiveDependency, kylinConf);
 logger.info("Hive Dependencies After Filtered: " + filteredHive);
 
 StringUtil.appendWithSeparator(kylinDependency, filteredHive);
@@ -259,13 +256,13 @@ public abstract class AbstractHadoopJob extends 
Configured implements Tool {
 }
 }
 
-private String filterKylinHiveDependency(String kylinHiveDependency) {
+private String filterKylinHiveDependency(String kylinHiveDependency, 
KylinConfig config) {
 if (StringUtils.isBlank(kylinHiveDependency))
 return "";
 
 StringBuilder jarList = new StringBuilder();
 
-Pattern hivePattern = 

[31/32] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[shaofengshi]

http://git-wip-us.apache.org/repos/asf/kylin/blob/3940e80f/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-if (admin.isTableEnabled(table)) {
-admin.disableTable(table);
+if 
(admin.isTableEnabled(TableName.valueOf(table))) {
+admin.disableTable(TableName.valueOf(table));
 }
-admin.deleteTable(table);
+admin.deleteTable(TableName.valueOf(table));
 logger.debug("Dropped htable: " + table);
 output.append("HBase table " + table + " is 
dropped. \n");
 } else {

http://git-wip-us.apache.org/repos/asf/kylin/blob/3940e80f/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index a150607..56f867a 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,9 +21,11 @@ package org.apache.kylin.storage.hbase.util;
 import java.io.IOException;
 
 import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.AbstractApplication;
 import org.apache.kylin.common.util.OptionsHelper;
 import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -38,8 +40,8 @@ public class CleanHtableCLI extends AbstractApplication {
 protected static final Logger logger = 
LoggerFactory.getLogger(CleanHtableCLI.class);
 
 private void clean() throws IOException {
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();
 
 for (HTableDescriptor descriptor : hbaseAdmin.listTables()) {
 String name = 

[07/32] kylin git commit: minor, fix concurrent issue with CubeSegment.cuboidBaseShards

[shaofengshi]

minor, fix concurrent issue with CubeSegment.cuboidBaseShards


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5d4d639d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5d4d639d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5d4d639d

Branch: refs/heads/v1.6.0-rc1-hbase1.x
Commit: 5d4d639ddb6b190e4978262e4e39778866b7270a
Parents: 8199a90
Author: gaodayue 
Authored: Wed Oct 26 21:39:11 2016 +0800
Committer: gaodayue 
Committed: Fri Oct 28 13:31:48 2016 +0800

--
 .../src/main/java/org/apache/kylin/cube/CubeSegment.java  | 7 +--
 1 file changed, 5 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/5d4d639d/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
index fdf1fb0..b3d6d6b 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
@@ -49,6 +49,8 @@ import 
com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
 
+import javax.annotation.concurrent.GuardedBy;
+
 @JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = 
Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = 
Visibility.NONE)
 public class CubeSegment implements Comparable, IBuildable, 
ISegment {
 
@@ -115,7 +117,8 @@ public class CubeSegment implements 
Comparable, IBuildable, ISegmen
 @JsonInclude(JsonInclude.Include.NON_EMPTY)
 private Map additionalInfo = new LinkedHashMap();
 
-private volatile Map cuboidBaseShards = 
Maps.newHashMap();//cuboid id ==> base(starting) shard for this cuboid
+@GuardedBy("this")
+private Map cuboidBaseShards = Maps.newHashMap(); // cuboid 
id ==> base(starting) shard for this cuboid
 
 public CubeDesc getCubeDesc() {
 return getCubeInstance().getDescriptor();
@@ -516,7 +519,7 @@ public class CubeSegment implements 
Comparable, IBuildable, ISegmen
 this.totalShards = totalShards;
 }
 
-public short getCuboidBaseShard(Long cuboidId) {
+public synchronized short getCuboidBaseShard(Long cuboidId) {
 if (totalShards > 0) {
 //shard squashed case
 

[21/32] kylin git commit: KYLIN-2143 allow more options from Extended Columns, COUNT_DISTINCT, RAW_TABLE

[shaofengshi]

KYLIN-2143 allow more options from Extended Columns,COUNT_DISTINCT,RAW_TABLE


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/67428a5d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/67428a5d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/67428a5d

Branch: refs/heads/v1.6.0-rc1-hbase1.x
Commit: 67428a5d83c6604b96d62da08edf48f844f02437
Parents: 8172d0b
Author: Jason 
Authored: Tue Nov 1 16:51:51 2016 +0800
Committer: Jason 
Committed: Tue Nov 1 16:52:31 2016 +0800

--
 webapp/app/js/controllers/cubeEdit.js  | 34 +++-
 webapp/app/partials/cubeDesigner/measures.html | 96 +++--
 2 files changed, 82 insertions(+), 48 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/67428a5d/webapp/app/js/controllers/cubeEdit.js
--
diff --git a/webapp/app/js/controllers/cubeEdit.js 
b/webapp/app/js/controllers/cubeEdit.js
index 5f46799..06d3120 100755
--- a/webapp/app/js/controllers/cubeEdit.js
+++ b/webapp/app/js/controllers/cubeEdit.js
@@ -137,18 +137,44 @@ KylinApp.controller('CubeEditCtrl', function ($scope, $q, 
$routeParams, $locatio
 return me_columns;
   };
 
-  $scope.getGroupByColumns = function () {
-var groupBy_columns=[];
+  $scope.getAllModelDimMeasureColumns = function () {
+var me_columns = [];
+if($scope.metaModel.model.metrics){
+  angular.forEach($scope.metaModel.model.metrics,function(metric,index){
+me_columns.push(metric);
+  })
+}
+
+
angular.forEach($scope.metaModel.model.dimensions,function(dimension,index){
+  if(dimension.columns){
+me_columns = me_columns.concat(dimension.columns);
+  }
+})
 
+return distinct_array(me_columns);
+  };
+
+  $scope.getAllModelDimColumns = function () {
+var me_columns = [];
 
angular.forEach($scope.metaModel.model.dimensions,function(dimension,index){
   if(dimension.columns){
-groupBy_columns = groupBy_columns.concat(dimension.columns);
+me_columns = me_columns.concat(dimension.columns);
   }
 })
 
-return groupBy_columns;
+return distinct_array(me_columns);
   };
 
+  function distinct_array(arrays){
+var arr = [];
+for(var item in arrays){
+  if(arr.indexOf(arrays[item])==-1){
+arr.push(arrays[item]);
+  }
+}
+return arr;
+  }
+
 
   $scope.getExtendedHostColumn = function(){
 var me_columns = [];

http://git-wip-us.apache.org/repos/asf/kylin/blob/67428a5d/webapp/app/partials/cubeDesigner/measures.html
--
diff --git a/webapp/app/partials/cubeDesigner/measures.html 
b/webapp/app/partials/cubeDesigner/measures.html
index b4513f2..c85bfd2 100755
--- a/webapp/app/partials/cubeDesigner/measures.html
+++ b/webapp/app/partials/cubeDesigner/measures.html
@@ -149,15 +149,22 @@
   
ng-init="newMeasure.function.parameter.value = 1">1
 
 
+ng-options="column as column for 
column in getAllModelDimMeasureColumns(newMeasure)" >
+  -- Select a Column 
--
+
+
   -- Select a Column 
--
 
 
   -- Select a Column 
--
@@ -165,44 +172,6 @@
   
   
   
-  
-  
-  
-  
-Return Type
-Maximum length of extended column  
-  
-  
-  
-  
-  
-
-  
-
-
-
-
-  {{newMeasure.function.returntype | uppercase}}
-  
-  
-  
-  
-
 
   
 
@@ -212,12 +181,51 @@
   
 
+ng-options="column as column for column in 
getAllModelDimColumns()" >
+  -- Select a Column --
+
+  
+
+  
+
+ 

[15/32] kylin git commit: KYLIN-2140 rename packaged js name

[shaofengshi]

KYLIN-2140 rename packaged js name


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/647d51e1
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/647d51e1
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/647d51e1

Branch: refs/heads/v1.6.0-rc1-hbase1.x
Commit: 647d51e1d5596774dcd4aab835f9ad4ffc81dd5a
Parents: 82e9907
Author: Jason 
Authored: Mon Oct 31 14:55:06 2016 +0800
Committer: Jason 
Committed: Mon Oct 31 14:55:06 2016 +0800

--
 build/script/build.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/647d51e1/build/script/build.sh
--
diff --git a/build/script/build.sh b/build/script/build.sh
index f54d41c..c68ca94 100755
--- a/build/script/build.sh
+++ b/build/script/build.sh
@@ -31,4 +31,4 @@ npm install -g bower   || { exit 1; }
 bower --allow-root install  || { exit 1; }
 npm install || { exit 1; }
 npm install -g grunt-cli|| { exit 1; }
-grunt dev --buildEnv=dev|| { exit 1; }
+grunt dev --buildEnv=dev --buildNumber=`date "+%Y%m%d%H%M%S"`  || { exit 1; }

[29/32] kylin git commit: KYLIN-1726 drop kafka intermediate file on build complete

[shaofengshi]

KYLIN-1726 drop kafka intermediate file on build complete

Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7d13664b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7d13664b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7d13664b

Branch: refs/heads/v1.6.0-rc1-hbase1.x
Commit: 7d13664b205492ea1c0c7caa2269c11f0867290c
Parents: 4a689b2
Author: shaofengshi 
Authored: Wed Nov 2 15:56:43 2016 +0800
Committer: shaofengshi 
Committed: Wed Nov 2 15:56:43 2016 +0800

--
 .../src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/7d13664b/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java
--
diff --git 
a/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java 
b/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java
index 99ebb6a..8695276 100644
--- a/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java
+++ b/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java
@@ -213,7 +213,6 @@ public class KafkaMRInput implements IMRInput {
 
 @Override
 protected ExecuteResult doWork(ExecutableContext context) throws 
ExecuteException {
-StringBuffer output = new StringBuffer();
 try {
 rmdirOnHDFS(getDataPath());
 } catch (IOException e) {
@@ -221,7 +220,7 @@ public class KafkaMRInput implements IMRInput {
 return new ExecuteResult(ExecuteResult.State.ERROR, 
e.getMessage());
 }
 
-return new ExecuteResult(ExecuteResult.State.SUCCEED, 
output.toString());
+return new ExecuteResult(ExecuteResult.State.SUCCEED, "HDFS path " 
+ getDataPath() + " is dropped.\n");
 }
 
 private void rmdirOnHDFS(String path) throws IOException {

[03/32] kylin git commit: port KYLIN-2068 to BeelineHiveClient(KYLIN-2125)

[shaofengshi]

port KYLIN-2068 to BeelineHiveClient(KYLIN-2125)


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/553d7c56
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/553d7c56
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/553d7c56

Branch: refs/heads/v1.6.0-rc1-hbase1.x
Commit: 553d7c56a0af49cb1c003f94543b4e600cd6f66e
Parents: 9ed775c
Author: Hongbin Ma 
Authored: Wed Oct 26 13:37:03 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Oct 27 08:34:11 2016 +0800

--
 .../org/apache/kylin/source/hive/BeelineHiveClient.java   | 10 +-
 .../java/org/apache/kylin/source/hive/CLIHiveClient.java  |  4 ++--
 .../apache/kylin/source/hive/HiveSourceTableLoader.java   |  1 +
 .../java/org/apache/kylin/source/hive/HiveTableMeta.java  |  6 --
 4 files changed, 12 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/553d7c56/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
--
diff --git 
a/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java 
b/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
index 0fbc39b..c8d56a5 100644
--- 
a/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
+++ 
b/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
@@ -115,7 +115,7 @@ public class BeelineHiveClient implements IHiveClient {
 
 List allColumns = 
Lists.newArrayList();
 while (columns.next()) {
-allColumns.add(new 
HiveTableMeta.HiveTableColumnMeta(columns.getString(4), columns.getString(6)));
+allColumns.add(new 
HiveTableMeta.HiveTableColumnMeta(columns.getString(4), columns.getString(6), 
columns.getString(12)));
 }
 builder.setAllColumns(allColumns);
 
@@ -139,7 +139,7 @@ public class BeelineHiveClient implements IHiveClient {
 if ("".equals(resultSet.getString(1).trim())) {
 break;
 }
-partitionColumns.add(new 
HiveTableMeta.HiveTableColumnMeta(resultSet.getString(1).trim(), 
resultSet.getString(2).trim()));
+partitionColumns.add(new 
HiveTableMeta.HiveTableColumnMeta(resultSet.getString(1).trim(), 
resultSet.getString(2).trim(), resultSet.getString(3).trim()));
 }
 builder.setPartitionColumns(partitionColumns);
 }
@@ -205,9 +205,9 @@ public class BeelineHiveClient implements IHiveClient {
 
 public static void main(String[] args) throws SQLException {
 
-//BeelineHiveClient loader = new BeelineHiveClient("-n root --hiveconf 
hive.security.authorization.sqlstd.confwhitelist.append='mapreduce.job.*|dfs.*' 
-u 'jdbc:hive2://sandbox:1'");
-BeelineHiveClient loader = new 
BeelineHiveClient(StringUtils.join(args, " "));
-HiveTableMeta hiveTableMeta = loader.getHiveTableMeta("default", 
"events");
+BeelineHiveClient loader = new BeelineHiveClient("-n root --hiveconf 
hive.security.authorization.sqlstd.confwhitelist.append='mapreduce.job.*|dfs.*' 
-u 'jdbc:hive2://sandbox:1'");
+//BeelineHiveClient loader = new 
BeelineHiveClient(StringUtils.join(args, " "));
+HiveTableMeta hiveTableMeta = loader.getHiveTableMeta("default", 
"test001");
 System.out.println(hiveTableMeta);
 loader.close();
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/553d7c56/source-hive/src/main/java/org/apache/kylin/source/hive/CLIHiveClient.java
--
diff --git 
a/source-hive/src/main/java/org/apache/kylin/source/hive/CLIHiveClient.java 
b/source-hive/src/main/java/org/apache/kylin/source/hive/CLIHiveClient.java
index ea74470..60cf47a 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/CLIHiveClient.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/CLIHiveClient.java
@@ -89,11 +89,11 @@ public class CLIHiveClient implements IHiveClient {
 List allColumns = 
Lists.newArrayList();
 List partitionColumns = 
Lists.newArrayList();
 for (FieldSchema fieldSchema : allFields) {
-allColumns.add(new 
HiveTableMeta.HiveTableColumnMeta(fieldSchema.getName(), 
fieldSchema.getType()));
+allColumns.add(new 
HiveTableMeta.HiveTableColumnMeta(fieldSchema.getName(), fieldSchema.getType(), 
fieldSchema.getComment()));
 }
 if (partitionFields != null && partitionFields.size() > 0) {
 for (FieldSchema fieldSchema : partitionFields) {
-partitionColumns.add(new 
HiveTableMeta.HiveTableColumnMeta(fieldSchema.getName(), 

[16/32] kylin git commit: KYLIN 1820 Column autocomplete should remove the user input in model designer

[shaofengshi]

KYLIN 1820 Column autocomplete should remove the user input in model designer

Signed-off-by: Jason 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5218c9f6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5218c9f6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5218c9f6

Branch: refs/heads/v1.6.0-rc1-hbase1.x
Commit: 5218c9f6bc6e27e86f29d3b75e5c6aaa73a9cb98
Parents: 647d51e
Author: chenzhx <346839...@qq.com>
Authored: Wed Oct 26 11:35:15 2016 +0800
Committer: Jason 
Committed: Mon Oct 31 16:59:15 2016 +0800

--
 webapp/app/js/directives/select.js | 939 
 1 file changed, 703 insertions(+), 236 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/5218c9f6/webapp/app/js/directives/select.js
--
diff --git a/webapp/app/js/directives/select.js 
b/webapp/app/js/directives/select.js
index c8cf6a2..7327af9 100644
--- a/webapp/app/js/directives/select.js
+++ b/webapp/app/js/directives/select.js
@@ -1,14 +1,13 @@
 /*!
  * ui-select
  * http://github.com/angular-ui/ui-select
- * Version: 0.13.2 - 2015-10-09T15:34:24.040Z
+ * Version: 0.19.5 - 2016-10-24T23:13:59.434Z
  * License: MIT
  */
 
 
 (function () {
 "use strict";
-
 var KEY = {
 TAB: 9,
 ENTER: 13,
@@ -42,7 +41,7 @@ var KEY = {
 return true;
 }
 
-if (e.metaKey) return true;
+if (e.metaKey || e.ctrlKey || e.altKey) return true;
 
 return false;
 },
@@ -55,6 +54,13 @@ var KEY = {
 },
 isHorizontalMovement: function (k){
   return ~[KEY.LEFT,KEY.RIGHT,KEY.BACKSPACE,KEY.DELETE].indexOf(k);
+},
+toSeparator: function (k) {
+  var sep = {ENTER:"\n",TAB:"\t",SPACE:" "}[k];
+  if (sep) return sep;
+  // return undefined for special keys other than enter, tab or space.
+  // no way to use them to cut strings.
+  return KEY[k] ? undefined : k;
 }
   };
 
@@ -103,11 +109,16 @@ var uis = angular.module('ui.select', [])
   placeholder: '', // Empty by default, like HTML tag 
   refreshDelay: 1000, // In milliseconds
   closeOnSelect: true,
+  skipFocusser: false,
   dropdownPosition: 'auto',
+  removeSelected: true,
+  resetSearchInput: true,
   generateId: function() {
 return latestId++;
   },
-  appendToBody: false
+  appendToBody: false,
+  spinnerEnabled: false,
+  spinnerClass: 'glyphicon-refresh ui-select-spin'
 })
 
 // See Rename minErr and make it accessible from outside 
https://github.com/angular/angular.js/issues/6913
@@ -139,11 +150,11 @@ var uis = angular.module('ui.select', [])
  */
 .filter('highlight', function() {
   function escapeRegexp(queryToEscape) {
-return queryToEscape.replace(/([.?*+^$[\]\\(){}|-])/g, '\\$1');
+return ('' + queryToEscape).replace(/([.?*+^$[\]\\(){}|-])/g, '\\$1');
   }
 
   return function(matchItem, query) {
-return query && matchItem ? matchItem.replace(new 
RegExp(escapeRegexp(query), 'gi'), '$&') : matchItem;
+return query && matchItem ? ('' + matchItem).replace(new 
RegExp(escapeRegexp(query), 'gi'), '$&') : matchItem;
   };
 })
 
@@ -169,8 +180,8 @@ var uis = angular.module('ui.select', [])
 }]);
 
 uis.directive('uiSelectChoices',
-  ['uiSelectConfig', 'uisRepeatParser', 'uiSelectMinErr', '$compile',
-  function(uiSelectConfig, RepeatParser, uiSelectMinErr, $compile) {
+  ['uiSelectConfig', 'uisRepeatParser', 'uiSelectMinErr', '$compile', 
'$window',
+  function(uiSelectConfig, RepeatParser, uiSelectMinErr, $compile, $window) {
 
   return {
 restrict: 'EA',
@@ -178,6 +189,9 @@ uis.directive('uiSelectChoices',
 replace: true,
 transclude: true,
 templateUrl: function(tElement) {
+  // Needed so the uiSelect can detect the transcluded content
+  tElement.addClass('ui-select-choices');
+
   // Gets theme attribute from parent (ui-select)
   var theme = tElement.parent().attr('theme') || uiSelectConfig.theme;
   return theme + '/choices.tpl.html';
@@ -187,44 +201,59 @@ uis.directive('uiSelectChoices',
 
   if (!tAttrs.repeat) throw uiSelectMinErr('repeat', "Expected 'repeat' 
expression.");
 
-  return function link(scope, element, attrs, $select, transcludeFn) {
+  // var repeat = RepeatParser.parse(attrs.repeat);
+  var groupByExp = tAttrs.groupBy;
+  var groupFilterExp = tAttrs.groupFilter;
 
-// var repeat = RepeatParser.parse(attrs.repeat);
-var groupByExp = attrs.groupBy;
-var groupFilterExp = attrs.groupFilter;
+  if (groupByExp) {
+var groups = tElement.querySelectorAll('.ui-select-choices-group');
+if (groups.length !== 1) throw uiSelectMinErr('rows', "Expected 1 
.ui-select-choices-group but got '{0}'.", groups.length);
+   

kylin git commit: view cube bug

[shaofengshi]

Repository: kylin
Updated Branches:
  refs/heads/v1.6.0-rc1 7d13664b2 -> 7a3793059


view cube bug

Signed-off-by: shaofengshi 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7a379305
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7a379305
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7a379305

Branch: refs/heads/v1.6.0-rc1
Commit: 7a3793059b5e790ba0de667b7feaabae7a468119
Parents: 7d13664
Author: luguosheng <550175...@qq.com>
Authored: Wed Nov 2 16:17:15 2016 +0800
Committer: shaofengshi 
Committed: Wed Nov 2 16:26:58 2016 +0800

--
 webapp/app/js/controllers/cubeEdit.js | 8 
 1 file changed, 4 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/7a379305/webapp/app/js/controllers/cubeEdit.js
--
diff --git a/webapp/app/js/controllers/cubeEdit.js 
b/webapp/app/js/controllers/cubeEdit.js
index 09dae15..acd4d32 100755
--- a/webapp/app/js/controllers/cubeEdit.js
+++ b/webapp/app/js/controllers/cubeEdit.js
@@ -293,11 +293,11 @@ KylinApp.controller('CubeEditCtrl', function ($scope, $q, 
$routeParams, $locatio
 });
 
 var queryParam = {
-  cube_name: $routeParams.cubeName
+  cubeId: $routeParams.cubeName
 };
-CubeService.list(queryParam, {},function(instance){
-  if (instance.length > 0) {
-$scope.instance = instance[0];
+CubeService.getCube(queryParam, {},function(instance){
+  if (instance) {
+$scope.instance = instance;
 $scope.state.cubeInstance =angular.toJson($scope.instance,true);
 
   } else {

kylin git commit: view cube bug

[shaofengshi]

Repository: kylin
Updated Branches:
  refs/heads/master 56e600ffa -> effb57edb


view cube bug

Signed-off-by: shaofengshi 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/effb57ed
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/effb57ed
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/effb57ed

Branch: refs/heads/master
Commit: effb57edb32a4c8599a742c0f30b37ae6f1c45c4
Parents: 56e600f
Author: luguosheng <550175...@qq.com>
Authored: Wed Nov 2 16:17:15 2016 +0800
Committer: shaofengshi 
Committed: Wed Nov 2 16:26:36 2016 +0800

--
 webapp/app/js/controllers/cubeEdit.js | 8 
 1 file changed, 4 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/effb57ed/webapp/app/js/controllers/cubeEdit.js
--
diff --git a/webapp/app/js/controllers/cubeEdit.js 
b/webapp/app/js/controllers/cubeEdit.js
index 09dae15..acd4d32 100755
--- a/webapp/app/js/controllers/cubeEdit.js
+++ b/webapp/app/js/controllers/cubeEdit.js
@@ -293,11 +293,11 @@ KylinApp.controller('CubeEditCtrl', function ($scope, $q, 
$routeParams, $locatio
 });
 
 var queryParam = {
-  cube_name: $routeParams.cubeName
+  cubeId: $routeParams.cubeName
 };
-CubeService.list(queryParam, {},function(instance){
-  if (instance.length > 0) {
-$scope.instance = instance[0];
+CubeService.getCube(queryParam, {},function(instance){
+  if (instance) {
+$scope.instance = instance;
 $scope.state.cubeInstance =angular.toJson($scope.instance,true);
 
   } else {

kylin git commit: KYLIN-2024 kylin TopN only support the first measure

[shaofengshi]

Repository: kylin
Updated Branches:
  refs/heads/yang21 f522e2bfe -> 37fbf0c1e


KYLIN-2024 kylin TopN only support the first measure


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/37fbf0c1
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/37fbf0c1
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/37fbf0c1

Branch: refs/heads/yang21
Commit: 37fbf0c1e22160b4da6ab88d59507a5e887d187c
Parents: f522e2b
Author: shaofengshi 
Authored: Wed Nov 2 13:20:10 2016 +0800
Committer: shaofengshi 
Committed: Wed Nov 2 16:07:13 2016 +0800

--
 .../kylin/measure/topn/TopNMeasureType.java  | 19 +--
 1 file changed, 13 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/37fbf0c1/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
 
b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
index 12343b3..3974a4f 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
@@ -39,8 +39,8 @@ import org.apache.kylin.metadata.datatype.DataTypeSerializer;
 import org.apache.kylin.metadata.model.FunctionDesc;
 import org.apache.kylin.metadata.model.MeasureDesc;
 import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.metadata.realization.SQLDigest;
 import 
org.apache.kylin.metadata.realization.CapabilityResult.CapabilityInfluence;
+import org.apache.kylin.metadata.realization.SQLDigest;
 import org.apache.kylin.metadata.tuple.Tuple;
 import org.apache.kylin.metadata.tuple.TupleInfo;
 import org.slf4j.Logger;
@@ -312,15 +312,16 @@ public class TopNMeasureType extends 
MeasureType {
 
 @Override
 public void adjustSqlDigest(List measureDescs, SQLDigest 
sqlDigest) {
+if (sqlDigest.aggregations.size() > 1) {
+return;
+}
+
 for (MeasureDesc measureDesc : measureDescs) {
 FunctionDesc topnFunc = measureDesc.getFunction();
 List topnLiteralCol = getTopNLiteralColumn(topnFunc);
 
-if (sqlDigest.groupbyColumns.containsAll(topnLiteralCol) == false)
-return;
-
-if (sqlDigest.aggregations.size() > 1) {
-return;
+if (sqlDigest.groupbyColumns.containsAll(topnLiteralCol) == false) 
{
+continue;
 }
 
 if (sqlDigest.aggregations.size() > 0) {
@@ -329,12 +330,18 @@ public class TopNMeasureType extends 
MeasureType {
 logger.warn("When query with topN, only SUM/Count function 
is allowed.");
 return;
 }
+
+if (isTopNCompatibleSum(measureDesc.getFunction(), origFunc) 
== false) {
+continue;
+}
+
 logger.info("Rewrite function " + origFunc + " to " + 
topnFunc);
 }
 
 sqlDigest.aggregations = Lists.newArrayList(topnFunc);
 sqlDigest.groupbyColumns.removeAll(topnLiteralCol);
 sqlDigest.metricColumns.addAll(topnLiteralCol);
+break;
 }
 }
 

[3/3] kylin git commit: KYLIN-1726 drop kafka intermediate file on build complete

[shaofengshi]

KYLIN-1726 drop kafka intermediate file on build complete

Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/56e600ff
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/56e600ff
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/56e600ff

Branch: refs/heads/master
Commit: 56e600ffa984e89a78907a63708a3a7c0fa10fd7
Parents: cdd633c
Author: shaofengshi 
Authored: Wed Nov 2 15:56:43 2016 +0800
Committer: shaofengshi 
Committed: Wed Nov 2 16:05:11 2016 +0800

--
 .../src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/56e600ff/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java
--
diff --git 
a/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java 
b/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java
index 99ebb6a..8695276 100644
--- a/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java
+++ b/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java
@@ -213,7 +213,6 @@ public class KafkaMRInput implements IMRInput {
 
 @Override
 protected ExecuteResult doWork(ExecutableContext context) throws 
ExecuteException {
-StringBuffer output = new StringBuffer();
 try {
 rmdirOnHDFS(getDataPath());
 } catch (IOException e) {
@@ -221,7 +220,7 @@ public class KafkaMRInput implements IMRInput {
 return new ExecuteResult(ExecuteResult.State.ERROR, 
e.getMessage());
 }
 
-return new ExecuteResult(ExecuteResult.State.SUCCEED, 
output.toString());
+return new ExecuteResult(ExecuteResult.State.SUCCEED, "HDFS path " 
+ getDataPath() + " is dropped.\n");
 }
 
 private void rmdirOnHDFS(String path) throws IOException {

[2/3] kylin git commit: KYLIN-2122 fix class visibility error

[shaofengshi]

KYLIN-2122 fix class visibility error


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/cdd633c9
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/cdd633c9
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/cdd633c9

Branch: refs/heads/master
Commit: cdd633c9ce224d0efc60aaffb3fac2162b86ebdd
Parents: dd1474b
Author: shaofengshi 
Authored: Wed Nov 2 13:58:29 2016 +0800
Committer: shaofengshi 
Committed: Wed Nov 2 16:05:04 2016 +0800

--
 .../src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/cdd633c9/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java
--
diff --git 
a/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java 
b/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java
index cdd7272..99ebb6a 100644
--- a/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java
+++ b/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java
@@ -208,7 +208,7 @@ public class KafkaMRInput implements IMRInput {
 }
 }
 
-static class GarbageCollectionStep extends AbstractExecutable {
+public static class GarbageCollectionStep extends AbstractExecutable {
 private static final Logger logger = 
LoggerFactory.getLogger(GarbageCollectionStep.class);
 
 @Override

[1/3] kylin git commit: KYLIN-2122 Move the partition offset calculation before submitting job

[shaofengshi]

Repository: kylin
Updated Branches:
  refs/heads/master 2bf5697da -> 56e600ffa


KYLIN-2122 Move the partition offset calculation before submitting job


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/dd1474be
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/dd1474be
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/dd1474be

Branch: refs/heads/master
Commit: dd1474be38cd33c1b2be349f85f340d0c629d074
Parents: 2bf5697
Author: shaofengshi 
Authored: Tue Oct 25 11:54:22 2016 +0800
Committer: shaofengshi 
Committed: Wed Nov 2 16:04:44 2016 +0800

--
 .../org/apache/kylin/cube/CubeInstance.java |   9 ++
 .../java/org/apache/kylin/cube/CubeManager.java | 103 +-
 .../org/apache/kylin/cube/CubeManagerTest.java  |   2 +-
 .../org/apache/kylin/cube/CubeSegmentsTest.java |   2 +-
 .../kylin/job/constant/ExecutableConstants.java |   1 +
 .../java/org/apache/kylin/source/ISource.java   |  13 ++-
 .../org/apache/kylin/source/SourceFactory.java  |   2 +-
 .../apache/kylin/source/SourcePartition.java| 103 ++
 .../kylin/engine/mr/JobBuilderSupport.java  |   2 +
 .../mr/steps/UpdateCubeInfoAfterBuildStep.java  |  67 
 .../kylin/provision/BuildCubeWithEngine.java|   8 +-
 .../kylin/provision/BuildCubeWithStream.java|  17 ++-
 .../apache/kylin/rest/service/JobService.java   |  16 +--
 .../apache/kylin/source/hive/HiveSource.java|  32 +-
 .../apache/kylin/source/kafka/KafkaMRInput.java |  90 ++--
 .../apache/kylin/source/kafka/KafkaSource.java  | 105 +-
 .../kylin/source/kafka/config/KafkaConfig.java  |   3 +
 .../source/kafka/hadoop/KafkaInputFormat.java   |  41 ---
 .../kylin/source/kafka/job/SeekOffsetStep.java  | 106 +--
 .../source/kafka/job/UpdateTimeRangeStep.java   |  78 +-
 .../kylin/source/kafka/util/KafkaClient.java|  24 -
 21 files changed, 483 insertions(+), 341 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/dd1474be/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
index 6bd3826..cb98991 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
@@ -446,6 +446,15 @@ public class CubeInstance extends RootPersistentEntity 
implements IRealization,
 return this.getDescriptor().getAutoMergeTimeRanges() != null && 
this.getDescriptor().getAutoMergeTimeRanges().length > 0;
 }
 
+public CubeSegment getLastSegment() {
+List existing = getSegments();
+if (existing.isEmpty()) {
+return null;
+} else {
+return existing.get(existing.size() - 1);
+}
+}
+
 @Override
 public int getSourceType() {
 return getFactTableDesc().getSourceType();

http://git-wip-us.apache.org/repos/asf/kylin/blob/dd1474be/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index a53849e..16b468f 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -34,8 +34,6 @@ import java.util.Random;
 import java.util.UUID;
 import java.util.concurrent.ConcurrentHashMap;
 
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Maps;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.KylinConfigExt;
@@ -68,9 +66,11 @@ import 
org.apache.kylin.metadata.realization.RealizationStatusEnum;
 import org.apache.kylin.metadata.realization.RealizationType;
 import org.apache.kylin.source.ReadableTable;
 import org.apache.kylin.source.SourceFactory;
+import org.apache.kylin.source.SourcePartition;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.google.common.base.Preconditions;
 import com.google.common.collect.HashMultimap;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Multimap;
@@ -434,52 +434,20 @@ public class CubeManager implements IRealizationProvider {
 
 // append a full build segment
 public CubeSegment appendSegment(CubeInstance cube) throws IOException {
-return appendSegment(cube, 0, 0, 0, 0, null, null);
+return appendSegment(cube, 0, Long.MAX_VALUE, 0, 0, 

kylin git commit: KYLIN-1726 drop kafka intermediate file on build complete

[shaofengshi]

Repository: kylin
Updated Branches:
  refs/heads/v1.6.0-rc1 4a689b2db -> 7d13664b2


KYLIN-1726 drop kafka intermediate file on build complete

Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7d13664b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7d13664b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7d13664b

Branch: refs/heads/v1.6.0-rc1
Commit: 7d13664b205492ea1c0c7caa2269c11f0867290c
Parents: 4a689b2
Author: shaofengshi 
Authored: Wed Nov 2 15:56:43 2016 +0800
Committer: shaofengshi 
Committed: Wed Nov 2 15:56:43 2016 +0800

--
 .../src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/7d13664b/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java
--
diff --git 
a/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java 
b/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java
index 99ebb6a..8695276 100644
--- a/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java
+++ b/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java
@@ -213,7 +213,6 @@ public class KafkaMRInput implements IMRInput {
 
 @Override
 protected ExecuteResult doWork(ExecutableContext context) throws 
ExecuteException {
-StringBuffer output = new StringBuffer();
 try {
 rmdirOnHDFS(getDataPath());
 } catch (IOException e) {
@@ -221,7 +220,7 @@ public class KafkaMRInput implements IMRInput {
 return new ExecuteResult(ExecuteResult.State.ERROR, 
e.getMessage());
 }
 
-return new ExecuteResult(ExecuteResult.State.SUCCEED, 
output.toString());
+return new ExecuteResult(ExecuteResult.State.SUCCEED, "HDFS path " 
+ getDataPath() + " is dropped.\n");
 }
 
 private void rmdirOnHDFS(String path) throws IOException {

kylin git commit: KYLIN-2122 fix class visibility error

[shaofengshi]

Repository: kylin
Updated Branches:
  refs/heads/v1.6.0-rc1 2beccbf94 -> 4a689b2db


KYLIN-2122 fix class visibility error


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4a689b2d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4a689b2d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4a689b2d

Branch: refs/heads/v1.6.0-rc1
Commit: 4a689b2db039aaa0dcc9cb3ce50192f11961b8a4
Parents: 2beccbf
Author: shaofengshi 
Authored: Wed Nov 2 13:58:29 2016 +0800
Committer: shaofengshi 
Committed: Wed Nov 2 13:58:29 2016 +0800

--
 .../src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/4a689b2d/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java
--
diff --git 
a/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java 
b/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java
index cdd7272..99ebb6a 100644
--- a/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java
+++ b/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java
@@ -208,7 +208,7 @@ public class KafkaMRInput implements IMRInput {
 }
 }
 
-static class GarbageCollectionStep extends AbstractExecutable {
+public static class GarbageCollectionStep extends AbstractExecutable {
 private static final Logger logger = 
LoggerFactory.getLogger(GarbageCollectionStep.class);
 
 @Override