kylin git commit: KYLIN-2105 quick review changes

[liyang]

Repository: kylin
Updated Branches:
  refs/heads/master db09f5f9c -> 047a23652


KYLIN-2105 quick review changes


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/047a2365
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/047a2365
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/047a2365

Branch: refs/heads/master
Commit: 047a23652f36012b5da03bb4c1cd57d079085717
Parents: db09f5f
Author: Li Yang 
Authored: Wed Oct 19 13:26:57 2016 +0800
Committer: Li Yang 
Committed: Wed Oct 19 13:26:57 2016 +0800

--
 .../src/main/java/org/apache/kylin/rest/service/QueryService.java | 3 +--
 .../main/java/org/apache/kylin/rest/util/QueryIdGenerator.java| 3 ++-
 server/src/main/resources/applicationContext.xml  | 1 -
 3 files changed, 3 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/047a2365/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java 
b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
index 2803125..42136f9 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -116,8 +116,7 @@ public class QueryService extends BasicService {
 private final String hbaseUrl;
 private final String userTableName;
 
-@Autowired
-private QueryIdGenerator queryIdGenerator;
+private QueryIdGenerator queryIdGenerator = new QueryIdGenerator();
 
 @Autowired
 private CacheManager cacheManager;

http://git-wip-us.apache.org/repos/asf/kylin/blob/047a2365/server-base/src/main/java/org/apache/kylin/rest/util/QueryIdGenerator.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/util/QueryIdGenerator.java 
b/server-base/src/main/java/org/apache/kylin/rest/util/QueryIdGenerator.java
index d9913e4..0156d46 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/util/QueryIdGenerator.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/util/QueryIdGenerator.java
@@ -40,6 +40,7 @@ public class QueryIdGenerator {
 postfix[i] = 
base26[ThreadLocalRandom.current().nextInt(base26.length)];
 }
 
-return String.format("%s_%s_%s", 
dateFormat.format(System.currentTimeMillis()), project, new String(postfix));
+return new String(postfix);
+//return String.format("%s_%s_%s", 
dateFormat.format(System.currentTimeMillis()), project, new String(postfix));
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/047a2365/server/src/main/resources/applicationContext.xml
--
diff --git a/server/src/main/resources/applicationContext.xml 
b/server/src/main/resources/applicationContext.xml
index c8a4e54..081dc53 100644
--- a/server/src/main/resources/applicationContext.xml
+++ b/server/src/main/resources/applicationContext.xml
@@ -39,7 +39,6 @@
 
 
 
-
 
 
 

[08/50] [abbrv] kylin git commit: Revert "KYLIN-2078 Can't see generated SQL at Web UI"

[lidong]

Revert "KYLIN-2078 Can't see generated SQL at Web UI"

This reverts commit 94a9f381548ff6ecfb253e189a3319252f18d16c.


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/dc969cd5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/dc969cd5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/dc969cd5

Branch: refs/heads/master-cdh5.7
Commit: dc969cd5cf7c37b1e0df6b8b6c620b9b9817754d
Parents: 94a9f38
Author: shaofengshi 
Authored: Mon Oct 10 17:54:08 2016 +0800
Committer: shaofengshi 
Committed: Mon Oct 10 17:54:08 2016 +0800

--
 .../java/org/apache/kylin/rest/controller/CubeController.java| 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/dc969cd5/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
 
b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index bb7dc6a..be242c3 100644
--- 
a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++ 
b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -40,6 +40,7 @@ import org.apache.kylin.engine.EngineFactory;
 import org.apache.kylin.job.JobInstance;
 import org.apache.kylin.job.JoinedFlatTable;
 import org.apache.kylin.metadata.model.IJoinedFlatTableDesc;
+import org.apache.kylin.metadata.model.SegmentStatusEnum;
 import org.apache.kylin.metadata.project.ProjectInstance;
 import org.apache.kylin.metadata.realization.RealizationStatusEnum;
 import org.apache.kylin.rest.exception.BadRequestException;
@@ -141,7 +142,8 @@ public class CubeController extends BasicController {
 @ResponseBody
 public GeneralResponse getSql(@PathVariable String cubeName, @PathVariable 
String segmentName) {
 CubeInstance cube = cubeService.getCubeManager().getCube(cubeName);
-IJoinedFlatTableDesc flatTableDesc = 
EngineFactory.getJoinedFlatTableDesc(cube.getDescriptor());
+CubeSegment cubeSegment = cube.getSegment(segmentName, 
SegmentStatusEnum.READY);
+IJoinedFlatTableDesc flatTableDesc = 
EngineFactory.getJoinedFlatTableDesc(cubeSegment);
 String sql = 
JoinedFlatTable.generateSelectDataStatement(flatTableDesc, false);
 
 GeneralResponse repsonse = new GeneralResponse();

[20/50] [abbrv] kylin git commit: KYLIN-2012 adjust rules for LookupTable & remove dead methods

[lidong]

KYLIN-2012 adjust rules for LookupTable & remove dead methods


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/36cf99ef
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/36cf99ef
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/36cf99ef

Branch: refs/heads/master-cdh5.7
Commit: 36cf99ef77486c1361a31f3e1f748bb277eca217
Parents: 96935fe
Author: gaodayue 
Authored: Thu Oct 13 16:56:05 2016 +0800
Committer: gaodayue 
Committed: Thu Oct 13 16:56:35 2016 +0800

--
 .../apache/kylin/metadata/MetadataManager.java  | 30 
 .../apache/kylin/source/hive/SchemaChecker.java | 27 ++
 2 files changed, 21 insertions(+), 36 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/36cf99ef/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java 
b/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
index 4e9c1ed..92d41b9 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
@@ -461,11 +461,6 @@ public class MetadataManager {
 reloadSourceTableAt(TableDesc.concatResourcePath(tableIdentity));
 }
 
-public void reloadTableCache(String tableIdentity) throws IOException {
-reloadSourceTableExt(tableIdentity);
-reloadSourceTable(tableIdentity);
-}
-
 public DataModelDesc getDataModelDesc(String name) {
 return dataModelDescMap.get(name);
 }
@@ -622,29 +617,4 @@ public class MetadataManager {
 srcTableExdMap.remove(tableIdentity);
 }
 
-public String appendDBName(String table) {
-
-if (table.indexOf(".") > 0)
-return table;
-
-Map map = getAllTablesMap();
-
-int count = 0;
-String result = null;
-for (TableDesc t : map.values()) {
-if (t.getName().equalsIgnoreCase(table)) {
-result = t.getIdentity();
-count++;
-}
-}
-
-if (count == 1)
-return result;
-
-if (count > 1) {
-logger.warn("There are more than 1 table named with '" + table + 
"' in different database; The program couldn't determine, randomly pick '" + 
result + "'");
-}
-return result;
-}
-
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/36cf99ef/source-hive/src/main/java/org/apache/kylin/source/hive/SchemaChecker.java
--
diff --git 
a/source-hive/src/main/java/org/apache/kylin/source/hive/SchemaChecker.java 
b/source-hive/src/main/java/org/apache/kylin/source/hive/SchemaChecker.java
index e955410..319ebee 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/SchemaChecker.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/SchemaChecker.java
@@ -152,7 +152,14 @@ public class SchemaChecker {
 }
 }
 
-private List checkAllUsedColumns(CubeInstance cube, TableDesc 
table, Map fieldsMap) {
+/**
+ * check whether all columns used in `cube` has compatible schema in 
current hive schema denoted by `fieldsMap`.
+ * @param cube cube to check, must use `table` in its model
+ * @param table kylin's table metadata
+ * @param fieldsMap current hive schema of `table`
+ * @return true if all columns used in `cube` has compatible schema with 
`fieldsMap`, false otherwise
+ */
+private List checkAllColumnsInCube(CubeInstance cube, TableDesc 
table, Map fieldsMap) {
 Set usedColumns = Sets.newHashSet();
 for (TblColRef col : cube.getAllColumns()) {
 usedColumns.add(col.getColumnDesc());
@@ -170,8 +177,15 @@ public class SchemaChecker {
 return violateColumns;
 }
 
-private boolean checkAllColumns(TableDesc table, List fields) 
{
-if (table.getColumnCount() != fields.size()) {
+/**
+ * check whether all columns in `table` are still in `fields` and have the 
same index as before.
+ *
+ * @param table kylin's table metadata
+ * @param fields current table metadata in hive
+ * @return true if only new columns are appended in hive, false otherwise
+ */
+private boolean checkAllColumnsInTableDesc(TableDesc table, 
List fields) {
+if (table.getColumnCount() > fields.size()) {
 return false;
 }
 
@@ -212,15 +226,16 @@ public class SchemaChecker {
 // if user reloads a fact table used by cube, then all used 

[27/50] [abbrv] kylin git commit: KYLIN-2088 Refactor MeasureType to allow mutliple UDAF defined on a measure type

[lidong]

KYLIN-2088 Refactor MeasureType to allow mutliple UDAF defined on a measure type


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0018a212
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0018a212
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0018a212

Branch: refs/heads/master-cdh5.7
Commit: 0018a212470bfe1e8edd3603d383da80f2ffd322
Parents: b4c970a
Author: Yang Li 
Authored: Sun Oct 16 23:44:49 2016 +0800
Committer: Yang Li 
Committed: Sun Oct 16 23:44:49 2016 +0800

--
 .../apache/kylin/common/KylinConfigBase.java|  1 -
 .../org/apache/kylin/measure/MeasureType.java   | 16 +++--
 .../kylin/measure/MeasureTypeFactory.java   | 46 +++---
 .../kylin/measure/basic/BasicMeasureType.java   |  6 --
 .../kylin/measure/bitmap/BitmapMeasureType.java | 43 -
 .../dim/DimCountDistinctMeasureType.java| 10 ++-
 .../ExtendedColumnMeasureType.java  |  6 +-
 .../kylin/measure/hllc/HLLCMeasureType.java | 10 ++-
 .../kylin/measure/raw/RawMeasureType.java   |  6 +-
 .../kylin/measure/topn/TopNMeasureType.java |  5 --
 .../kylin/metadata/realization/SQLDigest.java   | 43 -
 .../kylin/engine/mr/steps/CubeReducerTest.java  |  5 +-
 .../kylin/storage/hbase/ITStorageTest.java  |  3 +-
 .../kylin/query/relnode/ColumnRowType.java  |  7 +++
 .../kylin/query/relnode/OLAPAggregateRel.java   | 66 +++-
 .../apache/kylin/query/relnode/OLAPContext.java | 15 +++--
 .../kylin/query/schema/OLAPSchemaFactory.java   | 13 +++-
 .../cube/MeasureTypeOnlyAggrInBaseTest.java |  5 --
 18 files changed, 187 insertions(+), 119 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/0018a212/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 7dacd06..4942081 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -766,7 +766,6 @@ abstract public class KylinConfigBase implements 
Serializable {
 
 public Map getUDFs() {
 Map udfMap = getPropertiesByPrefix("kylin.query.udf.");
-udfMap.put("intersect_count", 
"org.apache.kylin.measure.bitmap.BitmapIntersectDistinctCountAggFunc");
 return udfMap;
 }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/0018a212/core-metadata/src/main/java/org/apache/kylin/measure/MeasureType.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureType.java 
b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureType.java
index e7312f2..de1b442 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureType.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureType.java
@@ -102,22 +102,20 @@ abstract public class MeasureType {
  * Query Rewrite
  * 
 */
 
-// TODO support user defined Calcite aggr function
-
 /** Whether or not Calcite rel-tree needs rewrite to do last around of 
aggregation */
 abstract public boolean needRewrite();
 
-/** Does the rewrite involves an extra field for the pre-calculated */
+/** Does the rewrite involves an extra field to hold the pre-calculated */
 public boolean needRewriteField() {
 return true;
 }
 
-/** Returns a Calcite aggregation function implementation class */
-abstract public Class getRewriteCalciteAggrFunctionClass();
-
-/** Some measure may return different class depends on call name, eg. 
BitmapMeasureType */
-public Class getRewriteCalciteAggrFunctionClass(String callName) {
-return getRewriteCalciteAggrFunctionClass();
+/**
+ * Returns a map from UDAF to Calcite aggregation function implementation 
class.
+ * There can be zero or more UDAF defined on a measure type.
+ */
+public Map getRewriteCalciteAggrFunctions() {
+return null;
 }
 
 /* 


http://git-wip-us.apache.org/repos/asf/kylin/blob/0018a212/core-metadata/src/main/java/org/apache/kylin/measure/MeasureTypeFactory.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureTypeFactory.java 

[25/50] [abbrv] kylin git commit: minor, move timezone setting early to surefire command line

[lidong]

minor, move timezone setting early to surefire command line


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/61a08d42
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/61a08d42
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/61a08d42

Branch: refs/heads/master-cdh5.7
Commit: 61a08d429a2c7419751b312e41bcee82048716e2
Parents: 021ccfd
Author: Yang Li 
Authored: Sun Oct 16 07:59:33 2016 +0800
Committer: Yang Li 
Committed: Sun Oct 16 07:59:33 2016 +0800

--
 .../src/test/java/org/apache/kylin/query/ITKylinQueryTest.java| 3 ---
 pom.xml   | 2 +-
 2 files changed, 1 insertion(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/61a08d42/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java 
b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
index 93d47f1..59a3a04 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
@@ -24,7 +24,6 @@ import java.io.File;
 import java.sql.SQLException;
 import java.util.List;
 import java.util.Map;
-import java.util.TimeZone;
 
 import org.apache.commons.lang3.StringUtils;
 import org.apache.kylin.common.KylinConfig;
@@ -66,8 +65,6 @@ public class ITKylinQueryTest extends KylinTestBase {
 
 
RemoveBlackoutRealizationsRule.blackList.add("CUBE[name=test_kylin_cube_with_view_left_join_empty]");
 
RemoveBlackoutRealizationsRule.blackList.add("CUBE[name=test_kylin_cube_with_view_inner_join_empty]");
-
-TimeZone.setDefault(TimeZone.getTimeZone("UTC"));
 }
 
 @AfterClass

http://git-wip-us.apache.org/repos/asf/kylin/blob/61a08d42/pom.xml
--
diff --git a/pom.xml b/pom.xml
index caa09ec..c7538e4 100644
--- a/pom.xml
+++ b/pom.xml
@@ -124,7 +124,7 @@
 
 
 
--Xms1G -Xmx2G -XX:MaxPermSize=512M
+-Xms1G -Xmx2G -XX:MaxPermSize=512M 
-Duser.timezone=UTC
 
 
 

[36/50] [abbrv] kylin git commit: KYLIN-2091 Add API to init the start-point (of each parition) for streaming cube

[lidong]

KYLIN-2091 Add API to init the start-point (of each parition) for streaming cube

Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c92f79ad
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c92f79ad
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c92f79ad

Branch: refs/heads/master-cdh5.7
Commit: c92f79ad39e562ed32fff1a55eb979f0593ed6e3
Parents: 5429006
Author: shaofengshi 
Authored: Mon Oct 17 17:40:57 2016 +0800
Committer: shaofengshi 
Committed: Mon Oct 17 17:53:56 2016 +0800

--
 .../apache/kylin/common/KylinConfigBase.java|  2 +-
 .../java/org/apache/kylin/cube/CubeSegment.java | 12 ++
 .../org/apache/kylin/cube/model/CubeDesc.java   | 13 ++
 .../kylin/measure/topn/TopNMeasureType.java |  2 +-
 server-base/pom.xml |  7 
 .../kylin/rest/controller/CubeController.java   | 10 ++---
 .../rest/controller/CubeDescController.java | 44 +++-
 .../kylin/source/kafka/job/SeekOffsetStep.java  |  8 +++-
 .../kylin/source/kafka/util/KafkaClient.java| 22 +-
 9 files changed, 108 insertions(+), 12 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/c92f79ad/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 4942081..73ac788 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -804,7 +804,7 @@ abstract public class KylinConfigBase implements 
Serializable {
 }
 
 public int getMaxBuildingSegments() {
-return Integer.parseInt(getOptional("kylin.cube.building.segment.max", 
"2"));
+return Integer.parseInt(getOptional("kylin.cube.building.segment.max", 
"10"));
 }
 
 public void setMaxBuildingSegments(int maxBuildingSegments) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/c92f79ad/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
index fdf1fb0..eb5b389 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
@@ -564,6 +564,12 @@ public class CubeSegment implements 
Comparable, IBuildable, ISegmen
 
 public void setSourcePartitionOffsetEnd(Map 
sourcePartitionOffsetEnd) {
 this.sourcePartitionOffsetEnd = sourcePartitionOffsetEnd;
+long total = 0;
+for (Long x : sourcePartitionOffsetEnd.values()) {
+total += x;
+}
+
+this.sourceOffsetEnd = total;
 }
 
 public Map getSourcePartitionOffsetStart() {
@@ -572,5 +578,11 @@ public class CubeSegment implements 
Comparable, IBuildable, ISegmen
 
 public void setSourcePartitionOffsetStart(Map 
sourcePartitionOffsetStart) {
 this.sourcePartitionOffsetStart = sourcePartitionOffsetStart;
+long total = 0;
+for (Long x : sourcePartitionOffsetStart.values()) {
+total += x;
+}
+
+this.sourceOffsetStart = total;
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/c92f79ad/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java 
b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index 4195451..bf3724a 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -170,6 +170,10 @@ public class CubeDesc extends RootPersistentEntity 
implements IEngineAware {
 
 private Map extendedColumnToHosts = 
Maps.newHashMap();
 
+@JsonProperty("partition_offset_start")
+@JsonInclude(JsonInclude.Include.NON_EMPTY)
+private Map partitionOffsetStart = Maps.newHashMap();
+
 public boolean isEnableSharding() {
 //in the future may extend to other storage that is shard-able
 return storageType != IStorageAware.ID_HBASE && storageType != 
IStorageAware.ID_HYBRID;
@@ -1011,6 +1015,14 @@ public class CubeDesc extends RootPersistentEntity 
implements IEngineAware {
 this.partitionDateEnd = partitionDateEnd;
 }
 
+public 

[38/50] [abbrv] kylin git commit: delete the blank line code

[lidong]

delete the blank line code

Signed-off-by: shaofengshi 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/cc2b59fa
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/cc2b59fa
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/cc2b59fa

Branch: refs/heads/master-cdh5.7
Commit: cc2b59faf1a12e76e6af83ba3a19fbe27ba54cc3
Parents: ac356f0
Author: lijieliang 
Authored: Fri Oct 14 14:28:01 2016 +0800
Committer: shaofengshi 
Committed: Mon Oct 17 18:25:26 2016 +0800

--
 .../src/main/java/org/apache/kylin/source/hive/HiveMRInput.java | 1 -
 1 file changed, 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/cc2b59fa/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
--
diff --git 
a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java 
b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
index 2ec1fbb..4ec8d3d 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
@@ -130,7 +130,6 @@ public class HiveMRInput implements IMRInput {
 @Override
 public void addStepPhase1_CreateFlatTable(DefaultChainedExecutable 
jobFlow) {
 final String cubeName = 
CubingExecutableUtil.getCubeName(jobFlow.getParams());
-
 final KylinConfig kylinConfig = 
CubeManager.getInstance(conf.getConfig()).getCube(cubeName).getConfig();
 
 String createFlatTableMethod = 
kylinConfig.getCreateFlatHiveTableMethod();

[14/50] [abbrv] kylin git commit: allow cleanupOldStorage() be overwritted

[lidong]

allow cleanupOldStorage() be overwritted


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4be729e0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4be729e0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4be729e0

Branch: refs/heads/master-cdh5.7
Commit: 4be729e0f7616b24cb289ead635b882abc515e1b
Parents: fad9e81
Author: shaofengshi 
Authored: Tue Oct 11 17:21:08 2016 +0800
Committer: shaofengshi 
Committed: Tue Oct 11 17:21:08 2016 +0800

--
 .../java/org/apache/kylin/provision/BuildCubeWithStream.java   | 6 +-
 1 file changed, 1 insertion(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/4be729e0/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java 
b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
index d2e46a5..2faa8d0 100644
--- a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
+++ b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
@@ -301,7 +301,7 @@ public class BuildCubeWithStream {
 HBaseMetadataTestCase.staticCleanupTestMetadata();
 }
 
-private static void cleanupOldStorage() throws Exception {
+protected void cleanupOldStorage() throws Exception {
 String[] args = { "--delete", "true" };
 StorageCleanupJob cli = new StorageCleanupJob();
 cli.execute(args);
@@ -321,10 +321,6 @@ public class BuildCubeWithStream {
 } catch (Throwable e) {
 logger.error("error", e);
 System.exit(1);
-} finally {
-if (buildCubeWithStream != null) {
-buildCubeWithStream.after();
-}
 }
 
 }

[37/50] [abbrv] kylin git commit: hive job use overrided MR job configuration by cube properties

[lidong]

hive job use overrided MR job configuration by cube properties

Signed-off-by: shaofengshi 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ac356f01
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ac356f01
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ac356f01

Branch: refs/heads/master-cdh5.7
Commit: ac356f014d52c4b13ad72e9d6a537e50e9ace5fb
Parents: c92f79a
Author: lijieliang 
Authored: Fri Oct 14 13:01:32 2016 +0800
Committer: shaofengshi 
Committed: Mon Oct 17 18:25:26 2016 +0800

--
 .../apache/kylin/source/hive/HiveMRInput.java   | 24 +---
 1 file changed, 21 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/ac356f01/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
--
diff --git 
a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java 
b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
index 202e480..2ec1fbb 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
@@ -35,6 +35,7 @@ import org.apache.kylin.common.util.CliCommandExecutor;
 import org.apache.kylin.common.util.Pair;
 import org.apache.kylin.cube.CubeInstance;
 import org.apache.kylin.cube.CubeManager;
+import org.apache.kylin.cube.CubeSegment;
 import org.apache.kylin.engine.mr.HadoopUtil;
 import org.apache.kylin.engine.mr.IMRInput;
 import org.apache.kylin.engine.mr.JobBuilderSupport;
@@ -60,6 +61,8 @@ import com.google.common.collect.Sets;
 
 public class HiveMRInput implements IMRInput {
 
+private static final String MR_OVERRIDE_JOB_QUEUENAME = 
"mapreduce.job.queuename";
+
 @Override
 public IMRBatchCubingInputSide 
getBatchCubingInputSide(IJoinedFlatTableDesc flatDesc) {
 return new BatchCubingInputSide(flatDesc);
@@ -154,7 +157,10 @@ public class HiveMRInput implements IMRInput {
 StringBuilder hiveInitBuf = new StringBuilder();
 hiveInitBuf.append("USE 
").append(conf.getConfig().getHiveDatabaseForIntermediateTable()).append(";\n");
 
hiveInitBuf.append(JoinedFlatTable.generateHiveSetStatements(conf));
-
+final KylinConfig kylinConfig = ((CubeSegment) 
flatTableDesc.getSegment()).getConfig();
+if 
(kylinConfig.getMRConfigOverride().get(MR_OVERRIDE_JOB_QUEUENAME) != null) {
+hiveInitBuf.append("SET 
mapreduce.job.queuename=").append(kylinConfig.getMRConfigOverride().get(MR_OVERRIDE_JOB_QUEUENAME)).append(";\n");
+}
 String rowCountOutputDir = 
JobBuilderSupport.getJobWorkingDir(conf, jobId) + "/row_count";
 
 RedistributeFlatHiveTableStep step = new 
RedistributeFlatHiveTableStep();
@@ -172,6 +178,11 @@ public class HiveMRInput implements IMRInput {
 final ShellExecutable step = new ShellExecutable();
 
 final HiveCmdBuilder hiveCmdBuilder = new HiveCmdBuilder();
+final KylinConfig kylinConfig = ((CubeSegment) 
flatTableDesc.getSegment()).getConfig();
+if 
(kylinConfig.getMRConfigOverride().get(MR_OVERRIDE_JOB_QUEUENAME) != null) {
+hiveCmdBuilder.addStatement("SET mapreduce.job.queuename =" +
+
kylinConfig.getMRConfigOverride().get(MR_OVERRIDE_JOB_QUEUENAME) + ";\n");
+}
 
hiveCmdBuilder.addStatement(JoinedFlatTable.generateHiveSetStatements(conf));
 hiveCmdBuilder.addStatement("set 
hive.exec.compress.output=false;\n");
 
hiveCmdBuilder.addStatement(JoinedFlatTable.generateCountDataStatement(flatTableDesc,
 rowCountOutputDir));
@@ -187,7 +198,7 @@ public class HiveMRInput implements IMRInput {
 
step.setName(ExecutableConstants.STEP_NAME_MATERIALIZE_HIVE_VIEW_IN_LOOKUP);
 HiveCmdBuilder hiveCmdBuilder = new HiveCmdBuilder();
 
-KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
+KylinConfig kylinConfig = ((CubeSegment) 
flatDesc.getSegment()).getConfig();
 MetadataManager metadataManager = 
MetadataManager.getInstance(kylinConfig);
 final Set lookupViewsTables = Sets.newHashSet();
 
@@ -201,6 +212,10 @@ public class HiveMRInput implements IMRInput {
 if (lookupViewsTables.size() == 0) {
 return null;
 }
+if 
(kylinConfig.getMRConfigOverride().get(MR_OVERRIDE_JOB_QUEUENAME) != null) {
+hiveCmdBuilder.addStatement("SET mapreduce.job.queuename =" +
+

[41/50] [abbrv] kylin git commit: KYLIN-2099 fix sample model

[lidong]

KYLIN-2099 fix sample model


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/03118bd4
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/03118bd4
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/03118bd4

Branch: refs/heads/master-cdh5.7
Commit: 03118bd445e7cfba14b75751fb9f478fe650b88a
Parents: c13ab1c
Author: Li Yang 
Authored: Tue Oct 18 12:36:43 2016 +0800
Committer: Li Yang 
Committed: Tue Oct 18 12:37:13 2016 +0800

--
 examples/sample_cube/template/model_desc/kylin_sales_model.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/03118bd4/examples/sample_cube/template/model_desc/kylin_sales_model.json
--
diff --git a/examples/sample_cube/template/model_desc/kylin_sales_model.json 
b/examples/sample_cube/template/model_desc/kylin_sales_model.json
index 13fc356..3c4fa5a 100644
--- a/examples/sample_cube/template/model_desc/kylin_sales_model.json
+++ b/examples/sample_cube/template/model_desc/kylin_sales_model.json
@@ -21,7 +21,7 @@
 "columns" : ["CAL_DT", "WEEK_BEG_DT"]
   }, {
 "table" : "DEFAULT.KYLIN_CATEGORY_GROUPINGS",
-"columns" : [ "USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3", "UPD_DATE", 
"UPD_USER", "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME" ]
+"columns" : [ "USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3", 
"META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME" ]
   }, {
 "table" : "DEFAULT.KYLIN_SALES",
 "columns" : [ "LSTG_FORMAT_NAME", "SELLER_ID", "PART_DT", "USER_ID", 
"REGION" ]

[30/50] [abbrv] kylin git commit: KYLIN-2097 Get 'Column does not exist in row key desc" on cube has TopN measure

[lidong]

KYLIN-2097 Get 'Column does not exist in row key desc" on cube has TopN measure

Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c3bfd770
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c3bfd770
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c3bfd770

Branch: refs/heads/master-cdh5.7
Commit: c3bfd770af93db7e7c295846898851c4bdc458c5
Parents: 70c85a7
Author: shaofengshi 
Authored: Mon Oct 17 12:22:45 2016 +0800
Committer: shaofengshi 
Committed: Mon Oct 17 12:23:11 2016 +0800

--
 .../java/org/apache/kylin/measure/topn/TopNMeasureType.java | 5 +
 1 file changed, 5 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/c3bfd770/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
 
b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
index 33ab314..b67c374 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
@@ -246,6 +246,11 @@ public class TopNMeasureType extends 
MeasureType {
 if (digest.groupbyColumns.containsAll(literalCol) == false)
 return null;
 
+for (TblColRef colRef : literalCol) {
+if (digest.filterColumns.contains(colRef) == true) {
+return null;
+}
+}
 unmatchedDimensions.removeAll(literalCol);
 unmatchedAggregations.remove(onlyFunction);
 return new CapabilityInfluence() {

[44/50] [abbrv] kylin git commit: KYLIN-1839, support kylin lib in HDFS

[lidong]

KYLIN-1839, support kylin lib in HDFS

Signed-off-by: terry 
Signed-off-by: Yang Li 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4473d710
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4473d710
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4473d710

Branch: refs/heads/master-cdh5.7
Commit: 4473d71011cc0e652eccf4f80269828caa5d3c73
Parents: d28835f
Author: terry 
Authored: Tue Oct 11 17:33:45 2016 +0800
Committer: Yang Li 
Committed: Wed Oct 19 07:30:13 2016 +0800

--
 .../engine/mr/common/AbstractHadoopJob.java | 29 +++-
 1 file changed, 16 insertions(+), 13 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/4473d710/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
index a138eec..bbb1711 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
@@ -46,6 +46,7 @@ import org.apache.hadoop.conf.Configured;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
 import org.apache.hadoop.mapreduce.Counters;
 import org.apache.hadoop.mapreduce.InputFormat;
 import org.apache.hadoop.mapreduce.InputSplit;
@@ -252,14 +253,10 @@ public abstract class AbstractHadoopJob extends 
Configured implements Tool {
 // for KylinJobMRLibDir
 String mrLibDir = kylinConf.getKylinJobMRLibDir();
 if (!StringUtils.isBlank(mrLibDir)) {
-File dirFileMRLIB = new File(mrLibDir);
-if (dirFileMRLIB.exists()) {
-if (kylinDependency.length() > 0)
-kylinDependency.append(",");
+if(kylinDependency.length() > 0) {
+kylinDependency.append(",");
+}
 kylinDependency.append(mrLibDir);
-} else {
-logger.info("The directory '" + mrLibDir + "' for 
'kylin.job.mr.lib.dir' does not exist!!!");
-}
 }
 
 setJobTmpJarsAndFiles(job, kylinDependency.toString());
@@ -300,21 +297,27 @@ public abstract class AbstractHadoopJob extends 
Configured implements Tool {
 try {
 Configuration jobConf = job.getConfiguration();
 FileSystem fs = FileSystem.getLocal(jobConf);
+FileSystem hdfs = FileSystem.get(jobConf);
 
 StringBuilder jarList = new StringBuilder();
 StringBuilder fileList = new StringBuilder();
 
 for (String fileName : fNameList) {
 Path p = new Path(fileName);
-if (fs.getFileStatus(p).isDirectory()) {
-appendTmpDir(job, fileName);
+FileSystem current = 
(fileName.startsWith(HdfsConstants.HDFS_URI_SCHEME) ? hdfs : fs);
+if(!current.exists(p)) {
+logger.warn("The directory '" + fileName + "for kylin 
dependency does not exist!!!");
+continue;
+}
+if (current.getFileStatus(p).isDirectory()) {
+appendTmpDir(job, current, fileName);
 continue;
 }
 
 StringBuilder list = (p.getName().endsWith(".jar")) ? jarList 
: fileList;
 if (list.length() > 0)
 list.append(",");
-list.append(fs.getFileStatus(p).getPath().toString());
+list.append(current.getFileStatus(p).getPath());
 }
 
 appendTmpFiles(fileList.toString(), jobConf);
@@ -324,13 +327,12 @@ public abstract class AbstractHadoopJob extends 
Configured implements Tool {
 }
 }
 
-private void appendTmpDir(Job job, String tmpDir) {
+private void appendTmpDir(Job job, FileSystem fs, String tmpDir) {
 if (StringUtils.isBlank(tmpDir))
 return;
 
 try {
 Configuration jobConf = job.getConfiguration();
-FileSystem fs = FileSystem.getLocal(jobConf);
 FileStatus[] fList = fs.listStatus(new Path(tmpDir));
 
 StringBuilder jarList = new StringBuilder();
@@ -339,7 +341,7 @@ public abstract class AbstractHadoopJob extends Configured 
implements Tool {
 for (FileStatus file : fList) {
 Path p = file.getPath();
 if 

[33/50] [abbrv] kylin git commit: minor, better name for hive build step

[lidong]

minor, better name for hive build step


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ef0c9e5c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ef0c9e5c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ef0c9e5c

Branch: refs/heads/master-cdh5.7
Commit: ef0c9e5c64d5fbb1aebc415755b8f2b9ff8d1a83
Parents: 44cf9fb
Author: Li Yang 
Authored: Mon Oct 17 13:38:42 2016 +0800
Committer: Li Yang 
Committed: Mon Oct 17 13:39:06 2016 +0800

--
 .../java/org/apache/kylin/job/constant/ExecutableConstants.java| 1 +
 .../src/main/java/org/apache/kylin/source/hive/HiveMRInput.java| 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/ef0c9e5c/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java
--
diff --git 
a/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java 
b/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java
index 893c034..ad0b1b1 100644
--- 
a/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java
+++ 
b/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java
@@ -51,6 +51,7 @@ public final class ExecutableConstants {
 public static final String STEP_NAME_SAVE_STATISTICS = "Save Cuboid 
Statistics";
 public static final String STEP_NAME_MERGE_CUBOID = "Merge Cuboid Data";
 public static final String STEP_NAME_UPDATE_CUBE_INFO = "Update Cube Info";
+public static final String STEP_NAME_HIVE_CLEANUP = "Hive Cleanup";
 public static final String STEP_NAME_GARBAGE_COLLECTION = "Garbage 
Collection";
 public static final String STEP_NAME_GARBAGE_COLLECTION_HDFS = "Garbage 
Collection on HDFS";
 public static final String STEP_NAME_BUILD_II = "Build Inverted Index";

http://git-wip-us.apache.org/repos/asf/kylin/blob/ef0c9e5c/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
--
diff --git 
a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java 
b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
index 09ac522..202e480 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
@@ -244,7 +244,7 @@ public class HiveMRInput implements IMRInput {
 @Override
 public void addStepPhase4_Cleanup(DefaultChainedExecutable jobFlow) {
 GarbageCollectionStep step = new GarbageCollectionStep();
-step.setName(ExecutableConstants.STEP_NAME_GARBAGE_COLLECTION);
+step.setName(ExecutableConstants.STEP_NAME_HIVE_CLEANUP);
 step.setIntermediateTableIdentity(getIntermediateTableIdentity());
 step.setExternalDataPath(JoinedFlatTable.getTableDir(flatDesc, 
JobBuilderSupport.getJobWorkingDir(conf, jobFlow.getId(;
 
step.setHiveViewIntermediateTableIdentities(hiveViewIntermediateTables);

[40/50] [abbrv] kylin git commit: KYLIN-2091 bug fix

[lidong]

KYLIN-2091 bug fix

Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c13ab1c3
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c13ab1c3
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c13ab1c3

Branch: refs/heads/master-cdh5.7
Commit: c13ab1c324bf6a56e75365e0e1ab5ecd48fd048a
Parents: 07e81fd
Author: shaofengshi 
Authored: Tue Oct 18 10:43:13 2016 +0800
Committer: shaofengshi 
Committed: Tue Oct 18 10:43:13 2016 +0800

--
 .../main/java/org/apache/kylin/cube/CubeSegment.java| 12 
 .../apache/kylin/source/kafka/job/SeekOffsetStep.java   |  2 ++
 2 files changed, 2 insertions(+), 12 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/c13ab1c3/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
index eb5b389..fdf1fb0 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
@@ -564,12 +564,6 @@ public class CubeSegment implements 
Comparable, IBuildable, ISegmen
 
 public void setSourcePartitionOffsetEnd(Map 
sourcePartitionOffsetEnd) {
 this.sourcePartitionOffsetEnd = sourcePartitionOffsetEnd;
-long total = 0;
-for (Long x : sourcePartitionOffsetEnd.values()) {
-total += x;
-}
-
-this.sourceOffsetEnd = total;
 }
 
 public Map getSourcePartitionOffsetStart() {
@@ -578,11 +572,5 @@ public class CubeSegment implements 
Comparable, IBuildable, ISegmen
 
 public void setSourcePartitionOffsetStart(Map 
sourcePartitionOffsetStart) {
 this.sourcePartitionOffsetStart = sourcePartitionOffsetStart;
-long total = 0;
-for (Long x : sourcePartitionOffsetStart.values()) {
-total += x;
-}
-
-this.sourceOffsetStart = total;
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/c13ab1c3/source-kafka/src/main/java/org/apache/kylin/source/kafka/job/SeekOffsetStep.java
--
diff --git 
a/source-kafka/src/main/java/org/apache/kylin/source/kafka/job/SeekOffsetStep.java
 
b/source-kafka/src/main/java/org/apache/kylin/source/kafka/job/SeekOffsetStep.java
index 2a3dbb5..a26f39d 100644
--- 
a/source-kafka/src/main/java/org/apache/kylin/source/kafka/job/SeekOffsetStep.java
+++ 
b/source-kafka/src/main/java/org/apache/kylin/source/kafka/job/SeekOffsetStep.java
@@ -116,6 +116,8 @@ public class SeekOffsetStep extends AbstractExecutable {
 }
 
 if (totalEndOffset > totalStartOffset) {
+segment.setSourceOffsetStart(totalStartOffset);
+segment.setSourceOffsetEnd(totalEndOffset);
 segment.setSourcePartitionOffsetStart(startOffsets);
 segment.setSourcePartitionOffsetEnd(endOffsets);
 segment.setName(CubeSegment.makeSegmentName(0, 0, 
totalStartOffset, totalEndOffset));

[12/50] [abbrv] kylin git commit: KYLIN-2076 Improve sample cube and data

[lidong]

KYLIN-2076 Improve sample cube and data

Signed-off-by: Li Yang 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0c7e2688
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0c7e2688
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0c7e2688

Branch: refs/heads/master-cdh5.7
Commit: 0c7e2688e80822cfbdad901cef909884bfa79ee8
Parents: 726b0ec
Author: Cheng Wang 
Authored: Mon Oct 10 17:49:33 2016 +0800
Committer: Li Yang 
Committed: Mon Oct 10 18:02:04 2016 +0800

--
 examples/sample_cube/create_sample_tables.sql   | 2 +
 .../sample_cube/data/DEFAULT.KYLIN_SALES.csv| 2 -
 .../cube_desc/kylin_sales_cube_desc.json|18 +-
 .../template/model_desc/kylin_sales_model.json  | 4 +-
 .../template/table/DEFAULT.KYLIN_SALES.json |10 +-
 5 files changed, 10030 insertions(+), 10004 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/0c7e2688/examples/sample_cube/create_sample_tables.sql
--
diff --git a/examples/sample_cube/create_sample_tables.sql 
b/examples/sample_cube/create_sample_tables.sql
index 5ed7b9b..8399ee8 100644
--- a/examples/sample_cube/create_sample_tables.sql
+++ b/examples/sample_cube/create_sample_tables.sql
@@ -181,6 +181,8 @@ TRANS_ID bigint
 ,PRICE decimal(19,4)
 ,ITEM_COUNT bigint
 ,SELLER_ID bigint
+,USER_ID string
+,REGION string
 )
 ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
 STORED AS TEXTFILE;

[42/50] [abbrv] kylin git commit: Ignore ITKylinQueryTest as it will be invoked by sub-classes

[lidong]

Ignore ITKylinQueryTest as it will be invoked by sub-classes

Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c84892f5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c84892f5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c84892f5

Branch: refs/heads/master-cdh5.7
Commit: c84892f527a6a16904d3cb801748e35248b4d90a
Parents: 03118bd
Author: shaofengshi 
Authored: Tue Oct 18 15:14:49 2016 +0800
Committer: shaofengshi 
Committed: Tue Oct 18 15:17:01 2016 +0800

--
 kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java | 1 +
 1 file changed, 1 insertion(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/c84892f5/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java 
b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
index ffd85e3..2ec5324 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
@@ -45,6 +45,7 @@ import org.junit.rules.ExpectedException;
 
 import com.google.common.collect.Maps;
 
+@Ignore("KylinQueryTest is contained by ITCombinationTest")
 public class ITKylinQueryTest extends KylinTestBase {
 
 @Rule

[46/50] [abbrv] kylin git commit: KYLIN-1839 minor refactor

[lidong]

KYLIN-1839 minor refactor


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6cafc422
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6cafc422
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6cafc422

Branch: refs/heads/master-cdh5.7
Commit: 6cafc422bc0ca2fa19a4516bcb4388719c99184c
Parents: aa1550c
Author: Yang Li 
Authored: Wed Oct 19 08:19:35 2016 +0800
Committer: Yang Li 
Committed: Wed Oct 19 08:30:49 2016 +0800

--
 .../kylin/engine/mr/common/AbstractHadoopJob.java  | 13 +++--
 1 file changed, 3 insertions(+), 10 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/6cafc422/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
index f70e3bb..a5b2d2e 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
@@ -318,7 +318,7 @@ public abstract class AbstractHadoopJob extends Configured 
implements Tool {
 }
 
 if (fs.getFileStatus(p).isDirectory()) {
-appendTmpDir(job, fs, p);
+appendTmpDir(job, fs, p, jarList, fileList);
 continue;
 }
 
@@ -335,18 +335,14 @@ public abstract class AbstractHadoopJob extends 
Configured implements Tool {
 }
 }
 
-private void appendTmpDir(Job job, FileSystem fs, Path tmpDir) {
+private void appendTmpDir(Job job, FileSystem fs, Path tmpDir, 
StringBuilder jarList, StringBuilder fileList) {
 try {
-Configuration jobConf = job.getConfiguration();
 FileStatus[] fList = fs.listStatus(tmpDir);
 
-StringBuilder jarList = new StringBuilder();
-StringBuilder fileList = new StringBuilder();
-
 for (FileStatus file : fList) {
 Path p = file.getPath();
 if (fs.getFileStatus(p).isDirectory()) {
-appendTmpDir(job, fs, p);
+appendTmpDir(job, fs, p, jarList, fileList);
 continue;
 }
 
@@ -356,9 +352,6 @@ public abstract class AbstractHadoopJob extends Configured 
implements Tool {
 list.append(fs.getFileStatus(p).getPath().toString());
 }
 
-appendTmpFiles(fileList.toString(), jobConf);
-appendTmpJars(jarList.toString(), jobConf);
-
 } catch (IOException e) {
 throw new RuntimeException(e);
 }

[05/50] [abbrv] kylin git commit: KYLIN-1726 minor update in kylin.sh

[lidong]

KYLIN-1726 minor update in kylin.sh


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7847f51c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7847f51c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7847f51c

Branch: refs/heads/master-cdh5.7
Commit: 7847f51cac4d8fe2be494ed20bc3616a17ad8757
Parents: ed643e6
Author: shaofengshi 
Authored: Mon Oct 10 15:19:09 2016 +0800
Committer: shaofengshi 
Committed: Mon Oct 10 15:19:09 2016 +0800

--
 build/bin/kylin.sh   |  2 +-
 .../apache/kylin/provision/BuildCubeWithStream.java  | 15 +++
 2 files changed, 8 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/7847f51c/build/bin/kylin.sh
--
diff --git a/build/bin/kylin.sh b/build/bin/kylin.sh
index 039be9f..da53d3d 100644
--- a/build/bin/kylin.sh
+++ b/build/bin/kylin.sh
@@ -40,7 +40,7 @@ function retrieveDependency() {
 
 export 
HBASE_CLASSPATH_PREFIX=${KYLIN_HOME}/conf:${KYLIN_HOME}/lib/*:${KYLIN_HOME}/tool/*:${KYLIN_HOME}/ext/*:${HBASE_CLASSPATH_PREFIX}
 export HBASE_CLASSPATH=${HBASE_CLASSPATH}:${hive_dependency}
-#export 
HBASE_CLASSPATH=${HBASE_CLASSPATH}:${hive_dependency}:${kafka_dependency}
+#export HBASE_CLASSPATH=${HBASE_CLASSPATH}:${kafka_dependency}
 }
 
 # start command

http://git-wip-us.apache.org/repos/asf/kylin/blob/7847f51c/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java 
b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
index a47fcde..65dd151 100644
--- a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
+++ b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
@@ -276,11 +276,6 @@ public class BuildCubeWithStream {
 
HBaseMetadataTestCase.staticCreateTestMetadata(HBaseMetadataTestCase.SANDBOX_TEST_DATA);
 }
 
-public static void afterClass() throws Exception {
-cleanupOldStorage();
-HBaseMetadataTestCase.staticCleanupTestMetadata();
-}
-
 public void after() {
 kafkaServer.stop();
 DefaultScheduler.destroyInstance();
@@ -301,6 +296,11 @@ public class BuildCubeWithStream {
 }
 }
 
+public void cleanup() throws Exception {
+cleanupOldStorage();
+HBaseMetadataTestCase.staticCleanupTestMetadata();
+}
+
 private static void cleanupOldStorage() throws Exception {
 String[] args = { "--delete", "true" };
 StorageCleanupJob cli = new StorageCleanupJob();
@@ -314,16 +314,15 @@ public class BuildCubeWithStream {
 buildCubeWithStream = new BuildCubeWithStream();
 buildCubeWithStream.before();
 buildCubeWithStream.build();
+logger.info("Build is done");
+buildCubeWithStream.cleanup();
 logger.info("Going to exit");
-System.exit(0);
 } catch (Throwable e) {
 logger.error("error", e);
-System.exit(1);
 } finally {
 if (buildCubeWithStream != null) {
 buildCubeWithStream.after();
 }
-afterClass();
 }
 
 }

[03/50] [abbrv] kylin git commit: KYLIN-2072 Cleanup old streaming code

[lidong]

http://git-wip-us.apache.org/repos/asf/kylin/blob/5aee0226/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/StreamingManager.java
--
diff --git 
a/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/StreamingManager.java
 
b/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/StreamingManager.java
deleted file mode 100644
index 271bf41..000
--- 
a/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/StreamingManager.java
+++ /dev/null
@@ -1,248 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.engine.streaming;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.ConcurrentHashMap;
-
-import org.apache.commons.lang3.StringUtils;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.persistence.JsonSerializer;
-import org.apache.kylin.common.persistence.ResourceStore;
-import org.apache.kylin.common.persistence.Serializer;
-import org.apache.kylin.metadata.MetadataConstants;
-import org.apache.kylin.metadata.cachesync.Broadcaster;
-import org.apache.kylin.metadata.cachesync.Broadcaster.Event;
-import org.apache.kylin.metadata.cachesync.CaseInsensitiveStringCache;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- */
-public class StreamingManager {
-
-private static final Logger logger = 
LoggerFactory.getLogger(StreamingManager.class);
-
-// static cached instances
-private static final ConcurrentHashMap 
CACHE = new ConcurrentHashMap();
-
-public static final Serializer STREAMING_SERIALIZER = new 
JsonSerializer(StreamingConfig.class);
-
-private KylinConfig config;
-
-// name ==> StreamingConfig
-private CaseInsensitiveStringCache streamingMap;
-
-public static void clearCache() {
-CACHE.clear();
-}
-
-private StreamingManager(KylinConfig config) throws IOException {
-this.config = config;
-this.streamingMap = new 
CaseInsensitiveStringCache(config, "streaming");
-
-// touch lower level metadata before registering my listener
-reloadAllStreaming();
-Broadcaster.getInstance(config).registerListener(new 
StreamingSyncListener(), "streaming");
-}
-
-private class StreamingSyncListener extends Broadcaster.Listener {
-@Override
-public void onClearAll(Broadcaster broadcaster) throws IOException {
-clearCache();
-}
-
-@Override
-public void onEntityChange(Broadcaster broadcaster, String entity, 
Event event, String cacheKey) throws IOException {
-if (event == Event.DROP)
-removeStreamingLocal(cacheKey);
-else
-reloadStreamingConfigLocal(cacheKey);
-}
-}
-
-private ResourceStore getStore() {
-return ResourceStore.getStore(this.config);
-}
-
-public static StreamingManager getInstance(KylinConfig config) {
-StreamingManager r = CACHE.get(config);
-if (r != null) {
-return r;
-}
-
-synchronized (StreamingManager.class) {
-r = CACHE.get(config);
-if (r != null) {
-return r;
-}
-try {
-r = new StreamingManager(config);
-CACHE.put(config, r);
-if (CACHE.size() > 1) {
-logger.warn("More than one singleton exist");
-}
-return r;
-} catch (IOException e) {
-throw new IllegalStateException("Failed to init 
StreamingManager from " + config, e);
-}
-}
-}
-
-private static String formatStreamingConfigPath(String name) {
-return ResourceStore.STREAMING_RESOURCE_ROOT + "/" + name + ".json";
-}
-
-private static String formatStreamingOutputPath(String streaming, int 
partition) {
-return ResourceStore.STREAMING_OUTPUT_RESOURCE_ROOT + "/" + streaming 
+ "_" + partition + ".json";
-}
-
-private static String 

[19/50] [abbrv] kylin git commit: KYLIN-2085 Add test cases

[lidong]

KYLIN-2085 Add test cases


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/96935fe9
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/96935fe9
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/96935fe9

Branch: refs/heads/master-cdh5.7
Commit: 96935fe6cd02efe0b8d11d25f2f67a83ad17
Parents: 8fbfc58
Author: lidongsjtu 
Authored: Wed Oct 12 17:21:24 2016 +0800
Committer: lidongsjtu 
Committed: Wed Oct 12 17:29:00 2016 +0800

--
 .../src/test/resources/query/sql/query100.sql   | 20 
 .../resources/query/sql_dynamic/query04.dat |  2 ++
 .../resources/query/sql_dynamic/query04.sql | 20 
 3 files changed, 42 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/96935fe9/kylin-it/src/test/resources/query/sql/query100.sql
--
diff --git a/kylin-it/src/test/resources/query/sql/query100.sql 
b/kylin-it/src/test/resources/query/sql/query100.sql
new file mode 100644
index 000..a2b0d1b
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql/query100.sql
@@ -0,0 +1,20 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+select count(*) as cnt from test_kylin_fact
+where lstg_format_name>='' and ''>=lstg_format_name
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/kylin/blob/96935fe9/kylin-it/src/test/resources/query/sql_dynamic/query04.dat
--
diff --git a/kylin-it/src/test/resources/query/sql_dynamic/query04.dat 
b/kylin-it/src/test/resources/query/sql_dynamic/query04.dat
new file mode 100644
index 000..e8506ab
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_dynamic/query04.dat
@@ -0,0 +1,2 @@
+
+
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/kylin/blob/96935fe9/kylin-it/src/test/resources/query/sql_dynamic/query04.sql
--
diff --git a/kylin-it/src/test/resources/query/sql_dynamic/query04.sql 
b/kylin-it/src/test/resources/query/sql_dynamic/query04.sql
new file mode 100644
index 000..135cf69
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_dynamic/query04.sql
@@ -0,0 +1,20 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+select count(*) as cnt from test_kylin_fact
+where lstg_format_name>=? and ?>=lstg_format_name
\ No newline at end of file

[06/50] [abbrv] kylin git commit: KYLIN-1923 update context columns

[lidong]

KYLIN-1923 update context columns

Signed-off-by: Li Yang 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e23f761e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e23f761e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e23f761e

Branch: refs/heads/master-cdh5.7
Commit: e23f761e7b516e77018f398af6cacab675dbf761
Parents: 7847f51
Author: Cheng Wang 
Authored: Sun Oct 9 17:15:51 2016 +0800
Committer: Li Yang 
Committed: Mon Oct 10 15:33:15 2016 +0800

--
 .../relnode/OLAPToEnumerableConverter.java  | 35 +---
 1 file changed, 31 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/e23f761e/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java
--
diff --git 
a/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java
 
b/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java
index 95ec617..55c27d3 100644
--- 
a/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java
+++ 
b/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java
@@ -19,6 +19,7 @@
 package org.apache.kylin.query.relnode;
 
 import java.util.List;
+import java.util.Set;
 
 import org.apache.calcite.adapter.enumerable.EnumerableRel;
 import org.apache.calcite.adapter.enumerable.EnumerableRelImplementor;
@@ -40,14 +41,18 @@ import org.apache.calcite.rel.type.RelDataType;
 import org.apache.calcite.sql.SqlExplainLevel;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.ClassUtil;
+import org.apache.kylin.metadata.filter.ColumnTupleFilter;
 import org.apache.kylin.metadata.filter.LogicalTupleFilter;
 import org.apache.kylin.metadata.filter.TupleFilter;
 import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
+import org.apache.kylin.metadata.model.TblColRef;
 import org.apache.kylin.metadata.realization.IRealization;
 import org.apache.kylin.query.routing.NoRealizationFoundException;
 import org.apache.kylin.query.routing.QueryRouter;
 import org.apache.kylin.query.schema.OLAPTable;
 
+import com.google.common.collect.Sets;
+
 /**
  */
 public class OLAPToEnumerableConverter extends ConverterImpl implements 
EnumerableRel {
@@ -90,7 +95,11 @@ public class OLAPToEnumerableConverter extends ConverterImpl 
implements Enumerab
 if (null != controllerCls && !controllerCls.isEmpty()) {
 OLAPContext.IAccessController accessController = 
(OLAPContext.IAccessController) ClassUtil.newInstance(controllerCls);
 TupleFilter tupleFilter = 
accessController.check(context.olapAuthen, context.allColumns, 
context.realization);
-context.filter = and(context.filter, tupleFilter);
+if (null != tupleFilter) {
+
context.filterColumns.addAll(collectColumns(tupleFilter));
+context.allColumns.addAll(collectColumns(tupleFilter));
+context.filter = and(context.filter, tupleFilter);
+}
 }
 }
 } catch (NoRealizationFoundException e) {
@@ -126,23 +135,41 @@ public class OLAPToEnumerableConverter extends 
ConverterImpl implements Enumerab
 return f2;
 if (f2 == null)
 return f1;
-
+
 if (f1.getOperator() == FilterOperatorEnum.AND) {
 f1.addChild(f2);
 return f1;
 }
-
+
 if (f2.getOperator() == FilterOperatorEnum.AND) {
 f2.addChild(f1);
 return f2;
 }
-
+
 LogicalTupleFilter and = new 
LogicalTupleFilter(FilterOperatorEnum.AND);
 and.addChild(f1);
 and.addChild(f2);
 return and;
 }
 
+private Set collectColumns(TupleFilter filter) {
+Set ret = Sets.newHashSet();
+collectColumnsRecursively(filter, ret);
+return ret;
+}
+
+private void collectColumnsRecursively(TupleFilter filter, Set 
collector) {
+if (filter == null)
+return;
+
+if (filter instanceof ColumnTupleFilter) {
+collector.add(((ColumnTupleFilter) filter).getColumn());
+}
+for (TupleFilter child : filter.getChildren()) {
+collectColumnsRecursively(child, collector);
+}
+}
+
 private Result buildHiveResult(EnumerableRelImplementor enumImplementor, 
Prefer pref, OLAPContext context) {
 RelDataType hiveRowType = getRowType();
 

[28/50] [abbrv] kylin git commit: KYLIN-2088 fix CI

[lidong]

KYLIN-2088 fix CI


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/81403c1b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/81403c1b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/81403c1b

Branch: refs/heads/master-cdh5.7
Commit: 81403c1b879f23760d91e2fc0864ece17da00d00
Parents: 0018a21
Author: Yang Li 
Authored: Mon Oct 17 05:54:45 2016 +0800
Committer: Yang Li 
Committed: Mon Oct 17 05:54:45 2016 +0800

--
 .../java/org/apache/kylin/query/ITKylinQueryTest.java | 14 --
 1 file changed, 8 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/81403c1b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java 
b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
index a0706ca..ffd85e3 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
@@ -248,6 +248,14 @@ public class ITKylinQueryTest extends KylinTestBase {
 }
 
 @Test
+public void testIntersectCountQuery() throws Exception {
+// cannot compare coz H2 does not support intersect count yet..
+if ("left".equalsIgnoreCase(joinType)) {
+this.batchExecuteQuery(getQueryFolderPrefix() + 
"src/test/resources/query/sql_intersect_count");
+}
+}
+
+@Test
 public void testDimDistinctCountQuery() throws Exception {
 execAndCompQuery(getQueryFolderPrefix() + 
"src/test/resources/query/sql_distinct_dim", null, true);
 }
@@ -355,10 +363,4 @@ public class ITKylinQueryTest extends KylinTestBase {
 this.batchExecuteQuery(getQueryFolderPrefix() + 
"src/test/resources/query/sql_window");
 }
 
-@Test
-public void testIntersectCountQuery() throws Exception {
-// cannot compare coz H2 does not support intersect count yet..
-this.batchExecuteQuery(getQueryFolderPrefix() + 
"src/test/resources/query/sql_intersect_count");
-}
-
 }

[04/50] [abbrv] kylin git commit: KYLIN-2072 Cleanup old streaming code

[lidong]

KYLIN-2072 Cleanup old streaming code

Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5aee0226
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5aee0226
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5aee0226

Branch: refs/heads/master-cdh5.7
Commit: 5aee022612c6fa40c41e8c00063714b79b6d5237
Parents: cb2b12b
Author: shaofengshi 
Authored: Sun Oct 9 13:10:50 2016 +0800
Committer: shaofengshi 
Committed: Mon Oct 10 13:32:44 2016 +0800

--
 assembly/pom.xml|   4 -
 .../kylin/job/streaming/KafkaDataLoader.java|  79 
 build/bin/cleanup_streaming_files.sh|  42 --
 build/bin/kylin.sh  |  61 ---
 build/bin/streaming_build.sh|  33 --
 build/bin/streaming_check.sh|  29 --
 build/bin/streaming_fillgap.sh  |  26 --
 build/bin/streaming_rolllog.sh  |  29 --
 .../metadata/streaming/StreamingConfig.java |  85 
 .../metadata/streaming/StreamingManager.java| 248 
 .../.settings/org.eclipse.core.resources.prefs  |   6 -
 .../.settings/org.eclipse.jdt.core.prefs| 386 ---
 .../.settings/org.eclipse.jdt.ui.prefs  |   7 -
 engine-streaming/pom.xml| 121 --
 .../kylin/engine/streaming/BootstrapConfig.java |  71 
 .../kylin/engine/streaming/IStreamingInput.java |  30 --
 .../engine/streaming/IStreamingOutput.java  |  34 --
 .../streaming/OneOffStreamingBuilder.java   |  71 
 .../engine/streaming/StreamingBatchBuilder.java |  43 ---
 .../kylin/engine/streaming/StreamingConfig.java |  85 
 .../engine/streaming/StreamingManager.java  | 248 
 .../kylin/engine/streaming/cli/MonitorCLI.java  |  88 -
 .../engine/streaming/cli/StreamingCLI.java  | 114 --
 .../streaming/cube/StreamingCubeBuilder.java| 168 
 .../diagnose/StreamingLogAnalyzer.java  |  96 -
 .../streaming/monitor/StreamingMonitor.java | 172 -
 .../engine/streaming/util/StreamingUtils.java   |  51 ---
 .../kylin/provision/BuildCubeWithStream.java|   4 +-
 pom.xml |   6 -
 .../rest/controller/StreamingController.java|   2 +-
 .../kylin/rest/controller/TableController.java  |   2 +-
 .../apache/kylin/rest/service/BasicService.java |   2 +-
 .../kylin/rest/service/StreamingService.java|   2 +-
 source-kafka/pom.xml|   6 -
 .../kafka/ByteBufferBackedInputStream.java  |  52 ---
 .../apache/kylin/source/kafka/KafkaSource.java  |   2 +-
 .../kylin/source/kafka/KafkaStreamingInput.java | 227 ---
 .../source/kafka/TimedJsonStreamParser.java |   1 +
 .../kafka/diagnose/KafkaInputAnalyzer.java  | 312 ---
 .../source/kafka/diagnose/KafkaVerify.java  | 101 -
 .../source/kafka/diagnose/TimeHistogram.java|  85 
 .../kafka/util/ByteBufferBackedInputStream.java |  52 +++
 .../kylin/source/kafka/util/KafkaRequester.java | 191 -
 .../kylin/source/kafka/util/KafkaUtils.java | 173 -
 .../test/java/TimedJsonStreamParserTest.java|   4 +-
 storage-hbase/pom.xml   |   4 -
 .../hbase/steps/HBaseStreamingOutput.java   |  98 -
 .../apache/kylin/tool/CubeMetaExtractor.java|   4 +-
 48 files changed, 397 insertions(+), 3360 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/5aee0226/assembly/pom.xml
--
diff --git a/assembly/pom.xml b/assembly/pom.xml
index 0c80afc..e6f83a8 100644
--- a/assembly/pom.xml
+++ b/assembly/pom.xml
@@ -47,10 +47,6 @@
 org.apache.kylin
 kylin-engine-mr
 
-
-org.apache.kylin
-kylin-engine-streaming
-
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/5aee0226/assembly/src/test/java/org/apache/kylin/job/streaming/KafkaDataLoader.java
--
diff --git 
a/assembly/src/test/java/org/apache/kylin/job/streaming/KafkaDataLoader.java 
b/assembly/src/test/java/org/apache/kylin/job/streaming/KafkaDataLoader.java
deleted file mode 100644
index 454f6cf..000
--- a/assembly/src/test/java/org/apache/kylin/job/streaming/KafkaDataLoader.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); 

[21/50] [abbrv] kylin git commit: KYLIN-2012 rename RealizationStatusEnum.DESCBROKEN to RealizationStatusEnum.BROKEN

[lidong]

KYLIN-2012 rename RealizationStatusEnum.DESCBROKEN to 
RealizationStatusEnum.BROKEN


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5974fc08
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5974fc08
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5974fc08

Branch: refs/heads/master-cdh5.7
Commit: 5974fc0870be17a8801c55c7496093d42dbb7c4f
Parents: 36cf99e
Author: gaodayue 
Authored: Thu Oct 13 17:16:02 2016 +0800
Committer: gaodayue 
Committed: Thu Oct 13 17:16:02 2016 +0800

--
 .../main/java/org/apache/kylin/cube/CubeDescManager.java  |  4 ++--
 .../src/main/java/org/apache/kylin/cube/CubeInstance.java |  2 +-
 .../src/main/java/org/apache/kylin/cube/CubeManager.java  |  6 +++---
 .../java/org/apache/kylin/cube/CubeManagerCacheTest.java  |  4 ++--
 .../kylin/metadata/realization/RealizationStatusEnum.java |  2 +-
 .../org/apache/kylin/rest/controller/CubeController.java  |  2 +-
 .../java/org/apache/kylin/rest/service/JobService.java|  2 +-
 webapp/app/partials/cubes/cubes.html  | 10 +-
 8 files changed, 16 insertions(+), 16 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/5974fc08/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
index d6364fe..602707f 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
@@ -158,10 +158,10 @@ public class CubeDescManager {
 cubeDescMap.putLocal(ndesc.getName(), ndesc);
 Cuboid.reloadCache(name);
 
-// if related cube is in DESCBROKEN state before, change it back to 
DISABLED
+// if related cube is in BROKEN state before, change it back to 
DISABLED
 CubeManager cubeManager = CubeManager.getInstance(config);
 for (CubeInstance cube : cubeManager.getCubesByDesc(name)) {
-if (cube.getStatus() == RealizationStatusEnum.DESCBROKEN) {
+if (cube.getStatus() == RealizationStatusEnum.BROKEN) {
 cubeManager.reloadCubeLocal(cube.getName());
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/5974fc08/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
index 720690d..b8e9394 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
@@ -153,7 +153,7 @@ public class CubeInstance extends RootPersistentEntity 
implements IRealization,
 // in a temporary broken state, so that user can edit and fix it. Broken 
state is often due to
 // schema changes at source.
 public boolean allowBrokenDescriptor() {
-return (getStatus() == RealizationStatusEnum.DISABLED  || getStatus() 
== RealizationStatusEnum.DESCBROKEN) && segments.isEmpty();
+return (getStatus() == RealizationStatusEnum.DISABLED  || getStatus() 
== RealizationStatusEnum.BROKEN) && segments.isEmpty();
 }
 
 public String getResourcePath() {

http://git-wip-us.apache.org/repos/asf/kylin/blob/5974fc08/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index c558c6d..1ebeb60 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -994,12 +994,12 @@ public class CubeManager implements IRealizationProvider {
 checkNotNull(cubeDesc, "cube descriptor '%s' (for cube '%s') not 
found", cube.getDescName(), cubeName);
 
 if (!cubeDesc.getError().isEmpty()) {
-cube.setStatus(RealizationStatusEnum.DESCBROKEN);
+cube.setStatus(RealizationStatusEnum.BROKEN);
 logger.warn("cube descriptor {} (for cube '{}') is broken", 
cubeDesc.getResourcePath(), cubeName);
 
-} else if (cube.getStatus() == RealizationStatusEnum.DESCBROKEN) {
+} else if (cube.getStatus() == RealizationStatusEnum.BROKEN) {
 cube.setStatus(RealizationStatusEnum.DISABLED);
-logger.info("cube {} changed from DESCBROKEN to DISABLED", 
cubeName);
+

[50/50] [abbrv] kylin git commit: KYLIN-1672 support kylin on cdh 5.7

[lidong]

KYLIN-1672 support kylin on cdh 5.7

Signed-off-by: Li Yang 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5f8b5e81
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5f8b5e81
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5f8b5e81

Branch: refs/heads/master-cdh5.7
Commit: 5f8b5e81614896dcd99bbb4b5fa106e764e101b8
Parents: b839156
Author: Lynne Jiang 
Authored: Mon May 16 03:33:27 2016 -0700
Committer: lidongsjtu 
Committed: Wed Oct 19 11:02:46 2016 +0800

--
 dev-support/test_all_against_hdp_2_2_4_2_2.sh   |   0
 .../kylin/engine/mr/steps/MockupMapContext.java |  15 +-
 examples/test_case_data/sandbox/core-site.xml   | 146 +++---
 examples/test_case_data/sandbox/hbase-site.xml  | 162 ++
 examples/test_case_data/sandbox/hdfs-site.xml   | 259 ++
 examples/test_case_data/sandbox/mapred-site.xml | 398 ++-
 examples/test_case_data/sandbox/yarn-site.xml   | 496 ++-
 pom.xml |  16 +-
 server/pom.xml  |  36 ++
 .../storage/hbase/steps/MockupMapContext.java   |  19 +-
 tool/pom.xml|  12 +
 11 files changed, 428 insertions(+), 1131 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/5f8b5e81/dev-support/test_all_against_hdp_2_2_4_2_2.sh
--
diff --git a/dev-support/test_all_against_hdp_2_2_4_2_2.sh 
b/dev-support/test_all_against_hdp_2_2_4_2_2.sh
old mode 100644
new mode 100755

http://git-wip-us.apache.org/repos/asf/kylin/blob/5f8b5e81/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
--
diff --git 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
index 847071d..9900465 100644
--- 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
+++ 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
@@ -77,6 +77,7 @@ public class MockupMapContext {
 outKV[0] = key;
 outKV[1] = value;
 }
+
 }
 
 @Override
@@ -99,6 +100,7 @@ public class MockupMapContext {
 throw new NotImplementedException();
 }
 
+
 @Override
 public float getProgress() {
 throw new NotImplementedException();
@@ -195,17 +197,17 @@ public class MockupMapContext {
 }
 
 @Override
-public RawComparator getSortComparator() {
+public boolean userClassesTakesPrecedence() {
 throw new NotImplementedException();
 }
 
 @Override
-public String getJar() {
+public RawComparator getSortComparator() {
 throw new NotImplementedException();
 }
 
 @Override
-public RawComparator getGroupingComparator() {
+public String getJar() {
 throw new NotImplementedException();
 }
 
@@ -221,7 +223,7 @@ public class MockupMapContext {
 
 @Override
 public boolean getProfileEnabled() {
-throw new NotImplementedException();
+return false;
 }
 
 @Override
@@ -308,6 +310,11 @@ public class MockupMapContext {
 public RawComparator getCombinerKeyGroupingComparator() {
 throw new NotImplementedException();
 }
+
+@Override
+public RawComparator getGroupingComparator() {
+return null;
+}
 });
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/5f8b5e81/examples/test_case_data/sandbox/core-site.xml
--
diff --git a/examples/test_case_data/sandbox/core-site.xml 
b/examples/test_case_data/sandbox/core-site.xml
index 9aa588c..6162406 100644
--- a/examples/test_case_data/sandbox/core-site.xml
+++ b/examples/test_case_data/sandbox/core-site.xml
@@ -14,152 +14,146 @@
   See the License for the specific language governing permissions and
   limitations under the License.
 -->
+
 
-
 
 fs.defaultFS
-hdfs://sandbox.hortonworks.com:8020
-true
+hdfs://quickstart.cloudera:8020
 
-
 
 fs.trash.interval
-360
+1
 
-
 
-
ha.failover-controller.active-standby-elector.zk.op.retries
-120
+io.compression.codecs
+

[35/50] [abbrv] kylin git commit: KYLIN-2099 fix sample table/cube desc

[lidong]

KYLIN-2099 fix sample table/cube desc


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5429006a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5429006a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5429006a

Branch: refs/heads/master-cdh5.7
Commit: 5429006a5976a52a08ef39976dfedfeb164629ea
Parents: 3924672
Author: Li Yang 
Authored: Mon Oct 17 16:42:44 2016 +0800
Committer: Li Yang 
Committed: Mon Oct 17 16:45:31 2016 +0800

--
 examples/sample_cube/create_sample_tables.sql   | 16 
 .../template/cube_desc/kylin_sales_cube_desc.json   |  2 +-
 .../table/DEFAULT.KYLIN_CATEGORY_GROUPINGS.json |  4 ++--
 3 files changed, 11 insertions(+), 11 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/5429006a/examples/sample_cube/create_sample_tables.sql
--
diff --git a/examples/sample_cube/create_sample_tables.sql 
b/examples/sample_cube/create_sample_tables.sql
index fa5e25b..9fa17d7 100644
--- a/examples/sample_cube/create_sample_tables.sql
+++ b/examples/sample_cube/create_sample_tables.sql
@@ -116,10 +116,10 @@ CAL_DT date COMMENT 'Date, PK'
 ,YTM_YN_ID tinyint
 ,YTQ_YN_ID tinyint
 ,YTW_YN_ID tinyint
-,CRE_DATE string
-,CRE_USER string
-,UPD_DATE string
-,UPD_USER string
+,KYLIN_CAL_DT_CRE_DATE string
+,KYLIN_CAL_DT_CRE_USER string
+,KYLIN_CAL_DT_UPD_DATE string
+,KYLIN_CAL_DT_UPD_USER string
 )
 COMMENT 'Date Dimension Table'
 ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
@@ -137,10 +137,10 @@ LEAF_CATEG_ID bigint COMMENT 'Category ID, PK'
 ,REGN_CATEG string
 ,USER_DEFINED_FIELD1 string COMMENT 'User Defined Field1'
 ,USER_DEFINED_FIELD3 string COMMENT 'User Defined Field3'
-,CRE_DATE string
-,UPD_DATE string COMMENT 'Last Updated Date'
-,CRE_USER string
-,UPD_USER string COMMENT 'Last Updated User'
+,KYLIN_GROUPINGS_CRE_DATE string
+,KYLIN_GROUPINGS_UPD_DATE string COMMENT 'Last Updated Date'
+,KYLIN_GROUPINGS_CRE_USER string
+,KYLIN_GROUPINGS_UPD_USER string COMMENT 'Last Updated User'
 ,META_CATEG_ID decimal
 ,META_CATEG_NAME string COMMENT 'Level1 Category'
 ,CATEG_LVL2_ID decimal

http://git-wip-us.apache.org/repos/asf/kylin/blob/5429006a/examples/sample_cube/template/cube_desc/kylin_sales_cube_desc.json
--
diff --git a/examples/sample_cube/template/cube_desc/kylin_sales_cube_desc.json 
b/examples/sample_cube/template/cube_desc/kylin_sales_cube_desc.json
index cc1cd76..3e9bff4 100644
--- a/examples/sample_cube/template/cube_desc/kylin_sales_cube_desc.json
+++ b/examples/sample_cube/template/cube_desc/kylin_sales_cube_desc.json
@@ -14,7 +14,7 @@
 "name" : "CATEGORY",
 "table" : "DEFAULT.KYLIN_CATEGORY_GROUPINGS",
 "column" : "{FK}",
-"derived" : [ "USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3", "UPD_DATE", 
"UPD_USER" ]
+"derived" : [ "USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3" ]
   }, {
 "name" : "CATEGORY_HIERARCHY",
 "table" : "DEFAULT.KYLIN_CATEGORY_GROUPINGS",

http://git-wip-us.apache.org/repos/asf/kylin/blob/5429006a/examples/sample_cube/template/table/DEFAULT.KYLIN_CATEGORY_GROUPINGS.json
--
diff --git 
a/examples/sample_cube/template/table/DEFAULT.KYLIN_CATEGORY_GROUPINGS.json 
b/examples/sample_cube/template/table/DEFAULT.KYLIN_CATEGORY_GROUPINGS.json
index 8e97309..accba35 100644
--- a/examples/sample_cube/template/table/DEFAULT.KYLIN_CATEGORY_GROUPINGS.json
+++ b/examples/sample_cube/template/table/DEFAULT.KYLIN_CATEGORY_GROUPINGS.json
@@ -40,7 +40,7 @@
 "datatype" : "string"
   }, {
 "id" : "10",
-"name" : "UPD_DATE",
+"name" : "KYLIN_GROUPINGS_UPD_DATE",
 "datatype" : "string"
   }, {
 "id" : "11",
@@ -48,7 +48,7 @@
 "datatype" : "string"
   }, {
 "id" : "12",
-"name" : "UPD_USER",
+"name" : "KYLIN_GROUPINGS_UPD_USER",
 "datatype" : "string"
   }, {
 "id" : "13",

[24/50] [abbrv] kylin git commit: minor, enhance KYLIN_HOME setup

[lidong]

minor, enhance KYLIN_HOME setup


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/021ccfd0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/021ccfd0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/021ccfd0

Branch: refs/heads/master-cdh5.7
Commit: 021ccfd0984bd41e2f2286ebec83efb551899575
Parents: bebbc78
Author: Yang Li 
Authored: Sat Oct 15 15:58:48 2016 +0800
Committer: Yang Li 
Committed: Sat Oct 15 15:59:38 2016 +0800

--
 build/bin/kylin.sh | 7 ---
 build/bin/metastore.sh | 9 +
 2 files changed, 9 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/021ccfd0/build/bin/kylin.sh
--
diff --git a/build/bin/kylin.sh b/build/bin/kylin.sh
index da53d3d..9286055e 100644
--- a/build/bin/kylin.sh
+++ b/build/bin/kylin.sh
@@ -17,9 +17,10 @@
 # limitations under the License.
 #
 
-# We should set KYLIN_HOME here for multiple tomcat instances that are on the 
same node.
-# In addition, we should set a KYLIN_HOME for the global use as normal.
-KYLIN_HOME=`dirname $0`/..
+dir=$(cd -P -- "$(dirname -- "$0")" && pwd -P)
+
+# set KYLIN_HOME with consideration for multiple instances that are on the 
same node
+KYLIN_HOME=${KYLIN_HOME:-"${dir}/../"}
 export KYLIN_HOME=`cd "$KYLIN_HOME"; pwd`
 dir="$KYLIN_HOME/bin"
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/021ccfd0/build/bin/metastore.sh
--
diff --git a/build/bin/metastore.sh b/build/bin/metastore.sh
index 5f34bff..baf3d7a 100755
--- a/build/bin/metastore.sh
+++ b/build/bin/metastore.sh
@@ -24,11 +24,12 @@
 # take a look at SandboxMetastoreCLI
 
 
-dir=$(dirname ${0})
+dir=$(cd -P -- "$(dirname -- "$0")" && pwd -P)
 
-# We should set KYLIN_HOME here for multiple tomcat instsances that are on the 
same node.
-# In addition, we should set a KYLIN_HOME for the global use as normal.
-export KYLIN_HOME=${dir}/../
+# set KYLIN_HOME with consideration for multiple instances that are on the 
same node
+KYLIN_HOME=${KYLIN_HOME:-"${dir}/../"}
+export KYLIN_HOME=`cd "$KYLIN_HOME"; pwd`
+dir="$KYLIN_HOME/bin"
 
 source ${dir}/check-env.sh
 

[07/50] [abbrv] kylin git commit: KYLIN-2078 Can't see generated SQL at Web UI

[lidong]

KYLIN-2078 Can't see generated SQL at Web UI


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/94a9f381
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/94a9f381
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/94a9f381

Branch: refs/heads/master-cdh5.7
Commit: 94a9f381548ff6ecfb253e189a3319252f18d16c
Parents: e23f761
Author: gaodayue 
Authored: Mon Oct 10 16:27:01 2016 +0800
Committer: gaodayue 
Committed: Mon Oct 10 16:27:01 2016 +0800

--
 .../java/org/apache/kylin/rest/controller/CubeController.java| 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/94a9f381/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
 
b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index be242c3..bb7dc6a 100644
--- 
a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++ 
b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -40,7 +40,6 @@ import org.apache.kylin.engine.EngineFactory;
 import org.apache.kylin.job.JobInstance;
 import org.apache.kylin.job.JoinedFlatTable;
 import org.apache.kylin.metadata.model.IJoinedFlatTableDesc;
-import org.apache.kylin.metadata.model.SegmentStatusEnum;
 import org.apache.kylin.metadata.project.ProjectInstance;
 import org.apache.kylin.metadata.realization.RealizationStatusEnum;
 import org.apache.kylin.rest.exception.BadRequestException;
@@ -142,8 +141,7 @@ public class CubeController extends BasicController {
 @ResponseBody
 public GeneralResponse getSql(@PathVariable String cubeName, @PathVariable 
String segmentName) {
 CubeInstance cube = cubeService.getCubeManager().getCube(cubeName);
-CubeSegment cubeSegment = cube.getSegment(segmentName, 
SegmentStatusEnum.READY);
-IJoinedFlatTableDesc flatTableDesc = 
EngineFactory.getJoinedFlatTableDesc(cubeSegment);
+IJoinedFlatTableDesc flatTableDesc = 
EngineFactory.getJoinedFlatTableDesc(cube.getDescriptor());
 String sql = 
JoinedFlatTable.generateSelectDataStatement(flatTableDesc, false);
 
 GeneralResponse repsonse = new GeneralResponse();

[15/50] [abbrv] kylin git commit: minor, controller classes extend from BasicController

[lidong]

minor, controller classes extend from BasicController


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ff851031
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ff851031
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ff851031

Branch: refs/heads/master-cdh5.7
Commit: ff8510313526134e504651b2d7879c1db8a8388f
Parents: 4be729e
Author: lidongsjtu 
Authored: Tue Oct 11 15:36:26 2016 +0800
Committer: lidongsjtu 
Committed: Tue Oct 11 17:28:13 2016 +0800

--
 .../java/org/apache/kylin/rest/controller/CubeDescController.java  | 2 +-
 .../main/java/org/apache/kylin/rest/controller/UserController.java | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/ff851031/server-base/src/main/java/org/apache/kylin/rest/controller/CubeDescController.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeDescController.java
 
b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeDescController.java
index 61584a3..4e595cd 100644
--- 
a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeDescController.java
+++ 
b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeDescController.java
@@ -36,7 +36,7 @@ import org.springframework.web.bind.annotation.ResponseBody;
  */
 @Controller
 @RequestMapping(value = "/cube_desc")
-public class CubeDescController {
+public class CubeDescController extends BasicController {
 
 @Autowired
 private CubeService cubeService;

http://git-wip-us.apache.org/repos/asf/kylin/blob/ff851031/server-base/src/main/java/org/apache/kylin/rest/controller/UserController.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/controller/UserController.java
 
b/server-base/src/main/java/org/apache/kylin/rest/controller/UserController.java
index bccfed4..b0336db 100644
--- 
a/server-base/src/main/java/org/apache/kylin/rest/controller/UserController.java
+++ 
b/server-base/src/main/java/org/apache/kylin/rest/controller/UserController.java
@@ -40,7 +40,7 @@ import org.springframework.web.bind.annotation.RequestMethod;
  */
 @Controller
 @RequestMapping(value = "/user")
-public class UserController {
+public class UserController extends BasicController {
 
 private static final Logger logger = 
LoggerFactory.getLogger(UserController.class);
 @Autowired

[49/50] [abbrv] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[lidong]

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b8391565
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b8391565
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b8391565

Branch: refs/heads/master-cdh5.7
Commit: b8391565e7855ea545c0a4fac72e0b4b5c06c872
Parents: db09f5f
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: lidongsjtu 
Committed: Wed Oct 19 10:48:12 2016 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 13 ++-
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 35 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  6 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 23 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 36 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 22 ++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 48 files changed, 403 insertions(+), 457 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/b8391565/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-etc.  Retry interval is a rough function based on 
hbase.client.pause.  At
-first we retry at this interval but then with backoff, we pretty 
quickly reach
-retrying every ten seconds.  See HConstants#RETRY_BACKOFF for how 
the backup
-ramps up.  Change 

[29/50] [abbrv] kylin git commit: The SweetAlert at the front page may out of the page if the content is too long

[lidong]

The SweetAlert at the front page may out of the page if the content is too long

Signed-off-by: Jason 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/70c85a7a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/70c85a7a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/70c85a7a

Branch: refs/heads/master-cdh5.7
Commit: 70c85a7a1279ed0246dbc3c61cc47275cd1b31c5
Parents: 81403c1
Author: liguohui 
Authored: Thu Oct 13 15:05:03 2016 +0800
Committer: Jason 
Committed: Mon Oct 17 11:20:01 2016 +0800

--
 webapp/app/less/component.less | 2 ++
 1 file changed, 2 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/70c85a7a/webapp/app/less/component.less
--
diff --git a/webapp/app/less/component.less b/webapp/app/less/component.less
index e23859d..ed1c9b7 100644
--- a/webapp/app/less/component.less
+++ b/webapp/app/less/component.less
@@ -1169,6 +1169,8 @@ ul.abn-tree li.abn-tree-row a {
 
 .sweet-alert .lead.text-muted {
   word-break: break-all;
+  max-height: 300px;
+  overflow: auto;
 }
 
 .modal-body.streaming-source .ace_editor {

[10/50] [abbrv] kylin git commit: KYLIN-2076 Improve sample cube and data

[lidong]

http://git-wip-us.apache.org/repos/asf/kylin/blob/0c7e2688/examples/sample_cube/template/cube_desc/kylin_sales_cube_desc.json
--
diff --git a/examples/sample_cube/template/cube_desc/kylin_sales_cube_desc.json 
b/examples/sample_cube/template/cube_desc/kylin_sales_cube_desc.json
index 99f8f46..cc1cd76 100644
--- a/examples/sample_cube/template/cube_desc/kylin_sales_cube_desc.json
+++ b/examples/sample_cube/template/cube_desc/kylin_sales_cube_desc.json
@@ -35,6 +35,16 @@
 "table" : "DEFAULT.KYLIN_SALES",
 "column" : "LSTG_FORMAT_NAME",
 "derived" : null
+  }, {
+"name" : "USER_ID",
+"table" : "DEFAULT.KYLIN_SALES",
+"column" : "USER_ID",
+"derived" : null
+  }, {
+"name" : "REGION",
+"table" : "DEFAULT.KYLIN_SALES",
+"column" : "REGION",
+"derived" : null
   } ],
   "measures" : [ {
 "name" : "GMV_SUM",
@@ -145,6 +155,12 @@
   "column" : "LSTG_FORMAT_NAME",
   "encoding" : "fixed_length:12"
 }, {
+  "column" : "USER_ID",
+  "encoding" : "dict"
+}, {
+  "column" : "REGION",
+  "encoding" : "dict"
+}, {
   "column" : "LSTG_SITE_ID",
   "encoding" : "dict"
 } ]
@@ -180,4 +196,4 @@
   "engine_type" : %default_engine_type%,
   "storage_type" : %default_storage_type%,
   "override_kylin_properties" : { }
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/0c7e2688/examples/sample_cube/template/model_desc/kylin_sales_model.json
--
diff --git a/examples/sample_cube/template/model_desc/kylin_sales_model.json 
b/examples/sample_cube/template/model_desc/kylin_sales_model.json
index 7ae47c1..13fc356 100644
--- a/examples/sample_cube/template/model_desc/kylin_sales_model.json
+++ b/examples/sample_cube/template/model_desc/kylin_sales_model.json
@@ -24,7 +24,7 @@
 "columns" : [ "USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3", "UPD_DATE", 
"UPD_USER", "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME" ]
   }, {
 "table" : "DEFAULT.KYLIN_SALES",
-"columns" : [ "LSTG_FORMAT_NAME", "SELLER_ID", "PART_DT" ]
+"columns" : [ "LSTG_FORMAT_NAME", "SELLER_ID", "PART_DT", "USER_ID", 
"REGION" ]
   } ],
   "metrics": [
 "PRICE",
@@ -40,4 +40,4 @@
 "partition_date_start" : 132537600,
 "partition_type" : "APPEND"
   }
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/0c7e2688/examples/sample_cube/template/table/DEFAULT.KYLIN_SALES.json
--
diff --git a/examples/sample_cube/template/table/DEFAULT.KYLIN_SALES.json 
b/examples/sample_cube/template/table/DEFAULT.KYLIN_SALES.json
index 4927343..2bccbeb 100644
--- a/examples/sample_cube/template/table/DEFAULT.KYLIN_SALES.json
+++ b/examples/sample_cube/template/table/DEFAULT.KYLIN_SALES.json
@@ -38,7 +38,15 @@
 "id" : "9",
 "name" : "SELLER_ID",
 "datatype" : "bigint"
+  }, {
+"id" : "10",
+"name" : "USER_ID",
+"datatype" : "string"
+  }, {
+"id" : "11",
+"name" : "REGION",
+"datatype" : "string"
   } ],
   "database" : "DEFAULT",
   "last_modified" : 0
-}
\ No newline at end of file
+}

[39/50] [abbrv] kylin git commit: KYLIN-2095 Allow cube to override Hive job configuration by properties

[lidong]

KYLIN-2095 Allow cube to override Hive job configuration by properties

Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/07e81fd0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/07e81fd0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/07e81fd0

Branch: refs/heads/master-cdh5.7
Commit: 07e81fd0b744e782d84383e327b1923cfc178d42
Parents: cc2b59f
Author: shaofengshi 
Authored: Mon Oct 17 22:07:57 2016 +0800
Committer: shaofengshi 
Committed: Mon Oct 17 22:07:57 2016 +0800

--
 .../apache/kylin/common/KylinConfigBase.java|  4 ++
 .../apache/kylin/source/hive/HiveMRInput.java   | 42 
 2 files changed, 30 insertions(+), 16 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/07e81fd0/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 73ac788..5a06813 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -250,6 +250,10 @@ abstract public class KylinConfigBase implements 
Serializable {
 return getPropertiesByPrefix("kylin.job.mr.config.override.");
 }
 
+public Map getHiveConfigOverride() {
+return getPropertiesByPrefix("kylin.hive.config.override.");
+}
+
 public String getKylinSparkJobJarPath() {
 final String jobJar = getOptional("kylin.job.jar.spark");
 if (StringUtils.isNotEmpty(jobJar)) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/07e81fd0/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
--
diff --git 
a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java 
b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
index 4ec8d3d..f3fceb1 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
@@ -20,6 +20,8 @@ package org.apache.kylin.source.hive;
 
 import java.io.IOException;
 import java.io.InputStream;
+import java.nio.charset.Charset;
+import java.util.Map;
 import java.util.Set;
 
 import org.apache.commons.io.IOUtils;
@@ -61,7 +63,6 @@ import com.google.common.collect.Sets;
 
 public class HiveMRInput implements IMRInput {
 
-private static final String MR_OVERRIDE_JOB_QUEUENAME = 
"mapreduce.job.queuename";
 
 @Override
 public IMRBatchCubingInputSide 
getBatchCubingInputSide(IJoinedFlatTableDesc flatDesc) {
@@ -157,9 +158,7 @@ public class HiveMRInput implements IMRInput {
 hiveInitBuf.append("USE 
").append(conf.getConfig().getHiveDatabaseForIntermediateTable()).append(";\n");
 
hiveInitBuf.append(JoinedFlatTable.generateHiveSetStatements(conf));
 final KylinConfig kylinConfig = ((CubeSegment) 
flatTableDesc.getSegment()).getConfig();
-if 
(kylinConfig.getMRConfigOverride().get(MR_OVERRIDE_JOB_QUEUENAME) != null) {
-hiveInitBuf.append("SET 
mapreduce.job.queuename=").append(kylinConfig.getMRConfigOverride().get(MR_OVERRIDE_JOB_QUEUENAME)).append(";\n");
-}
+appendHiveOverrideProperties(kylinConfig, hiveInitBuf);
 String rowCountOutputDir = 
JobBuilderSupport.getJobWorkingDir(conf, jobId) + "/row_count";
 
 RedistributeFlatHiveTableStep step = new 
RedistributeFlatHiveTableStep();
@@ -178,10 +177,7 @@ public class HiveMRInput implements IMRInput {
 
 final HiveCmdBuilder hiveCmdBuilder = new HiveCmdBuilder();
 final KylinConfig kylinConfig = ((CubeSegment) 
flatTableDesc.getSegment()).getConfig();
-if 
(kylinConfig.getMRConfigOverride().get(MR_OVERRIDE_JOB_QUEUENAME) != null) {
-hiveCmdBuilder.addStatement("SET mapreduce.job.queuename =" +
-
kylinConfig.getMRConfigOverride().get(MR_OVERRIDE_JOB_QUEUENAME) + ";\n");
-}
+appendHiveOverrideProperties2(kylinConfig, hiveCmdBuilder);
 
hiveCmdBuilder.addStatement(JoinedFlatTable.generateHiveSetStatements(conf));
 hiveCmdBuilder.addStatement("set 
hive.exec.compress.output=false;\n");
 
hiveCmdBuilder.addStatement(JoinedFlatTable.generateCountDataStatement(flatTableDesc,
 rowCountOutputDir));
@@ -192,6 +188,7 @@ public class HiveMRInput implements IMRInput {
 return step;
 }
 
+
 public ShellExecutable 

[01/50] [abbrv] kylin git commit: KYLIN-1726 package rename [Forced Update!]

[lidong]

Repository: kylin
Updated Branches:
  refs/heads/master-cdh5.7 d5efd44c0 -> 5f8b5e816 (forced update)


KYLIN-1726 package rename


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c67fa740
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c67fa740
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c67fa740

Branch: refs/heads/master-cdh5.7
Commit: c67fa740d364d372c7a6424fd160570cc7e890c4
Parents: 5aee022
Author: shaofengshi 
Authored: Sun Oct 9 13:24:59 2016 +0800
Committer: shaofengshi 
Committed: Mon Oct 10 13:32:44 2016 +0800

--
 .../apache/kylin/source/kafka/KafkaMRInput.java |   3 +
 .../kylin/source/kafka/MergeOffsetStep.java |  80 ---
 .../kylin/source/kafka/SeekOffsetStep.java  | 140 --
 .../source/kafka/StringStreamingParser.java |  51 ---
 .../apache/kylin/source/kafka/TopicMeta.java|  46 --
 .../kylin/source/kafka/UpdateTimeRangeStep.java | 117 ---
 .../kylin/source/kafka/job/MergeOffsetStep.java |  80 +++
 .../kylin/source/kafka/job/SeekOffsetStep.java  | 141 +++
 .../source/kafka/job/UpdateTimeRangeStep.java   | 117 +++
 .../kafka/util/ByteBufferBackedInputStream.java |   6 +-
 10 files changed, 344 insertions(+), 437 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/c67fa740/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java
--
diff --git 
a/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java 
b/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java
index 6358ee1..4d1f5c9 100644
--- a/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java
+++ b/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java
@@ -43,6 +43,9 @@ import org.apache.kylin.metadata.model.IJoinedFlatTableDesc;
 import org.apache.kylin.metadata.model.TableDesc;
 import org.apache.kylin.metadata.model.TblColRef;
 import org.apache.kylin.source.kafka.config.KafkaConfig;
+import org.apache.kylin.source.kafka.job.MergeOffsetStep;
+import org.apache.kylin.source.kafka.job.SeekOffsetStep;
+import org.apache.kylin.source.kafka.job.UpdateTimeRangeStep;
 
 import javax.annotation.Nullable;
 import java.io.IOException;

http://git-wip-us.apache.org/repos/asf/kylin/blob/c67fa740/source-kafka/src/main/java/org/apache/kylin/source/kafka/MergeOffsetStep.java
--
diff --git 
a/source-kafka/src/main/java/org/apache/kylin/source/kafka/MergeOffsetStep.java 
b/source-kafka/src/main/java/org/apache/kylin/source/kafka/MergeOffsetStep.java
deleted file mode 100644
index 18c959a..000
--- 
a/source-kafka/src/main/java/org/apache/kylin/source/kafka/MergeOffsetStep.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-package org.apache.kylin.source.kafka;
-
-import java.io.IOException;
-import java.util.Collections;
-import java.util.List;
-
-import org.apache.kylin.cube.CubeInstance;
-import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.cube.CubeSegment;
-import org.apache.kylin.cube.CubeUpdate;
-import org.apache.kylin.engine.mr.steps.CubingExecutableUtil;
-import org.apache.kylin.job.exception.ExecuteException;
-import org.apache.kylin.job.execution.AbstractExecutable;
-import org.apache.kylin.job.execution.ExecutableContext;
-import org.apache.kylin.job.execution.ExecuteResult;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- */
-public class MergeOffsetStep extends AbstractExecutable {
-
-private static final Logger logger = 
LoggerFactory.getLogger(MergeOffsetStep.class);
-public MergeOffsetStep() {
-super();
-}
-
-@Override
-protected ExecuteResult doWork(ExecutableContext context) throws 
ExecuteException {
-final CubeManager cubeManager = 

[23/50] [abbrv] kylin git commit: Revert "KYLIN-2012 rename RealizationStatusEnum.DESCBROKEN to RealizationStatusEnum.BROKEN"

[lidong]

Revert "KYLIN-2012 rename RealizationStatusEnum.DESCBROKEN to 
RealizationStatusEnum.BROKEN"

This reverts commit 5974fc0870be17a8801c55c7496093d42dbb7c4f.


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/bebbc784
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/bebbc784
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/bebbc784

Branch: refs/heads/master-cdh5.7
Commit: bebbc78476c6b41b50fc0492134a5418d522533e
Parents: e6698c8
Author: shaofengshi 
Authored: Fri Oct 14 11:46:01 2016 +0800
Committer: shaofengshi 
Committed: Fri Oct 14 11:46:01 2016 +0800

--
 .../main/java/org/apache/kylin/cube/CubeDescManager.java  |  4 ++--
 .../src/main/java/org/apache/kylin/cube/CubeInstance.java |  2 +-
 .../src/main/java/org/apache/kylin/cube/CubeManager.java  |  6 +++---
 .../java/org/apache/kylin/cube/CubeManagerCacheTest.java  |  4 ++--
 .../kylin/metadata/realization/RealizationStatusEnum.java |  2 +-
 .../org/apache/kylin/rest/controller/CubeController.java  |  2 +-
 .../java/org/apache/kylin/rest/service/JobService.java|  2 +-
 webapp/app/partials/cubes/cubes.html  | 10 +-
 8 files changed, 16 insertions(+), 16 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/bebbc784/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
index 602707f..d6364fe 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
@@ -158,10 +158,10 @@ public class CubeDescManager {
 cubeDescMap.putLocal(ndesc.getName(), ndesc);
 Cuboid.reloadCache(name);
 
-// if related cube is in BROKEN state before, change it back to 
DISABLED
+// if related cube is in DESCBROKEN state before, change it back to 
DISABLED
 CubeManager cubeManager = CubeManager.getInstance(config);
 for (CubeInstance cube : cubeManager.getCubesByDesc(name)) {
-if (cube.getStatus() == RealizationStatusEnum.BROKEN) {
+if (cube.getStatus() == RealizationStatusEnum.DESCBROKEN) {
 cubeManager.reloadCubeLocal(cube.getName());
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/bebbc784/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
index b8e9394..720690d 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
@@ -153,7 +153,7 @@ public class CubeInstance extends RootPersistentEntity 
implements IRealization,
 // in a temporary broken state, so that user can edit and fix it. Broken 
state is often due to
 // schema changes at source.
 public boolean allowBrokenDescriptor() {
-return (getStatus() == RealizationStatusEnum.DISABLED  || getStatus() 
== RealizationStatusEnum.BROKEN) && segments.isEmpty();
+return (getStatus() == RealizationStatusEnum.DISABLED  || getStatus() 
== RealizationStatusEnum.DESCBROKEN) && segments.isEmpty();
 }
 
 public String getResourcePath() {

http://git-wip-us.apache.org/repos/asf/kylin/blob/bebbc784/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index 1ebeb60..c558c6d 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -994,12 +994,12 @@ public class CubeManager implements IRealizationProvider {
 checkNotNull(cubeDesc, "cube descriptor '%s' (for cube '%s') not 
found", cube.getDescName(), cubeName);
 
 if (!cubeDesc.getError().isEmpty()) {
-cube.setStatus(RealizationStatusEnum.BROKEN);
+cube.setStatus(RealizationStatusEnum.DESCBROKEN);
 logger.warn("cube descriptor {} (for cube '{}') is broken", 
cubeDesc.getResourcePath(), cubeName);
 
-} else if (cube.getStatus() == RealizationStatusEnum.BROKEN) {
+} else if (cube.getStatus() == RealizationStatusEnum.DESCBROKEN) {
 cube.setStatus(RealizationStatusEnum.DISABLED);
-  

[31/50] [abbrv] kylin git commit: KYLIN-2084 Unload sample table failed

[lidong]

KYLIN-2084 Unload sample table failed

Signed-off-by: Jason 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ef44b7f6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ef44b7f6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ef44b7f6

Branch: refs/heads/master-cdh5.7
Commit: ef44b7f6873b2e5ac55243818877474cdc0376d8
Parents: c3bfd77
Author: Jason 
Authored: Mon Oct 17 13:18:57 2016 +0800
Committer: Jason 
Committed: Mon Oct 17 13:20:16 2016 +0800

--
 .../java/org/apache/kylin/metadata/MetadataManager.java   | 10 ++
 .../org/apache/kylin/rest/controller/TableController.java |  3 +++
 .../java/org/apache/kylin/rest/service/ModelService.java  |  7 +++
 3 files changed, 20 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/ef44b7f6/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java 
b/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
index 92d41b9..e0c78ce 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
@@ -496,6 +496,16 @@ public class MetadataManager {
 return false;
 }
 
+public List getModelsUsingTable(String tableName, String 
projectName) throws IOException {
+List models = new ArrayList<>();
+for (DataModelDesc modelDesc : getModels(projectName)) {
+if (modelDesc.getAllTables().contains(tableName.toUpperCase())) {
+models.add(modelDesc.getName());
+}
+}
+return models;
+}
+
 public boolean isTableInAnyModel(String tableName) {
 for (DataModelDesc modelDesc : getModels()) {
 if (modelDesc.getAllTables().contains(tableName.toUpperCase())) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/ef44b7f6/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
 
b/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
index 34cc57f..d9050c1 100644
--- 
a/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
+++ 
b/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
@@ -197,6 +197,9 @@ public class TableController extends BasicController {
 if (!modelService.isTableInModel(tableName, project)) {
 cubeMgmtService.removeTableFromProject(tableName, project);
 rtn = true;
+}else{
+List models = 
modelService.getModelsUsingTable(tableName, project);
+throw new InternalErrorException("Table is already in use by 
models "+models);
 }
 } catch (IOException e) {
 logger.error(e.getMessage(), e);

http://git-wip-us.apache.org/repos/asf/kylin/blob/ef44b7f6/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java 
b/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java
index bfff619..2f8667c 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java
@@ -132,4 +132,11 @@ public class ModelService extends BasicService {
 tableName = dbTableName[0] + "." + dbTableName[1];
 return getMetadataManager().isTableInModel(tableName, projectName);
 }
+
+@PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#desc, 
'ADMINISTRATION') or hasPermission(#desc, 'MANAGEMENT')")
+public List getModelsUsingTable(String tableName, String 
projectName) throws IOException {
+String[] dbTableName = HadoopUtil.parseHiveTableName(tableName);
+tableName = dbTableName[0] + "." + dbTableName[1];
+return getMetadataManager().getModelsUsingTable(tableName, 
projectName);
+}
 }

[02/50] [abbrv] kylin git commit: KYLIN-2072 Cleanup old streaming code

[lidong]

http://git-wip-us.apache.org/repos/asf/kylin/blob/5aee0226/source-kafka/src/main/java/org/apache/kylin/source/kafka/diagnose/TimeHistogram.java
--
diff --git 
a/source-kafka/src/main/java/org/apache/kylin/source/kafka/diagnose/TimeHistogram.java
 
b/source-kafka/src/main/java/org/apache/kylin/source/kafka/diagnose/TimeHistogram.java
deleted file mode 100644
index 1c579c6..000
--- 
a/source-kafka/src/main/java/org/apache/kylin/source/kafka/diagnose/TimeHistogram.java
+++ /dev/null
@@ -1,85 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.source.kafka.diagnose;
-
-import java.util.concurrent.atomic.AtomicLong;
-
-public class TimeHistogram {
-private long[] bucketsBoundary;
-private AtomicLong[] counters;
-private String id;
-
-private static Object printLock = new Object();
-
-/**
- * example: [10,20] will generate three  buckets: (-∞,10), 
[10,20),[20,+∞)
- * unit: second
- */
-public TimeHistogram(long[] bucketsBoundary, String id) {
-this.bucketsBoundary = bucketsBoundary;
-this.counters = new AtomicLong[this.bucketsBoundary.length + 1];
-for (int i = 0; i < counters.length; i++) {
-this.counters[i] = new AtomicLong();
-}
-this.id = id;
-}
-
-/**
- * @param second in seconds
- */
-public void process(long second) {
-for (int i = 0; i < bucketsBoundary.length; ++i) {
-if (second < bucketsBoundary[i]) {
-counters[i].incrementAndGet();
-return;
-}
-}
-
-counters[bucketsBoundary.length].incrementAndGet();
-}
-
-/**
- * @param millis in milli seconds
- */
-public void processMillis(long millis) {
-process(millis / 1000);
-}
-
-public void printStatus() {
-long[] countersSnapshot = new long[counters.length];
-for (int i = 0; i < countersSnapshot.length; i++) {
-countersSnapshot[i] = counters[i].get();
-}
-
-long sum = 0;
-for (long counter : countersSnapshot) {
-sum += counter;
-}
-
-synchronized (printLock) {
-System.out.println("== status of TimeHistogram " + id 
+ " =");
-
-for (int i = 0; i < countersSnapshot.length; ++i) {
-System.out.println(String.format("bucket: %d , count: %d 
,percentage: %.4f", i, countersSnapshot[i], 1.0 * countersSnapshot[i] / (sum == 
0 ? 1 : sum)));
-}
-
-}
-}
-
-}

http://git-wip-us.apache.org/repos/asf/kylin/blob/5aee0226/source-kafka/src/main/java/org/apache/kylin/source/kafka/util/ByteBufferBackedInputStream.java
--
diff --git 
a/source-kafka/src/main/java/org/apache/kylin/source/kafka/util/ByteBufferBackedInputStream.java
 
b/source-kafka/src/main/java/org/apache/kylin/source/kafka/util/ByteBufferBackedInputStream.java
new file mode 100644
index 000..7a42598
--- /dev/null
+++ 
b/source-kafka/src/main/java/org/apache/kylin/source/kafka/util/ByteBufferBackedInputStream.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+package org.apache.kylin.source.kafka.util;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.ByteBuffer;
+
+/**
+ */

[17/50] [abbrv] kylin git commit: KYLIN-2068: add optional comment column when sync hive table columns

[lidong]

KYLIN-2068: add optional comment column when sync hive table columns

Signed-off-by: shaofengshi 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/58afe010
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/58afe010
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/58afe010

Branch: refs/heads/master-cdh5.7
Commit: 58afe0103098dd0845cb50a0751e3305a973f834
Parents: 460536c
Author: Yiming Liu 
Authored: Mon Oct 10 23:49:19 2016 +0800
Committer: shaofengshi 
Committed: Wed Oct 12 09:25:24 2016 +0800

--
 .../apache/kylin/metadata/model/ColumnDesc.java | 13 ++
 examples/sample_cube/create_sample_tables.sql   | 49 +++-
 .../source/hive/HiveSourceTableLoader.java  |  1 +
 webapp/app/js/model/tableConfig.js  |  3 +-
 webapp/app/partials/tables/table_detail.html|  4 ++
 5 files changed, 46 insertions(+), 24 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/58afe010/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
index 257cb3b..d131b92 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
@@ -20,6 +20,7 @@ package org.apache.kylin.metadata.model;
 
 import java.io.Serializable;
 
+import com.fasterxml.jackson.annotation.JsonInclude;
 import org.apache.commons.lang.StringUtils;
 import org.apache.kylin.metadata.datatype.DataType;
 
@@ -51,6 +52,9 @@ public class ColumnDesc implements Serializable {
 private String name;
 @JsonProperty("datatype")
 private String datatype;
+@JsonProperty("comment")
+@JsonInclude(JsonInclude.Include.NON_NULL)
+private String comment;
 
 // parsed from data type
 private DataType type;
@@ -109,6 +113,14 @@ public class ColumnDesc implements Serializable {
 this.table = table;
 }
 
+public String getComment() {
+return comment;
+}
+
+public void setComment(String comment) {
+this.comment = comment;
+}
+
 public DataType getType() {
 return type;
 }
@@ -195,6 +207,7 @@ public class ColumnDesc implements Serializable {
 "id='" + id + '\'' +
 ", name='" + name + '\'' +
 ", datatype='" + datatype + '\'' +
+", comment='" + comment + '\'' +
 '}';
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/58afe010/examples/sample_cube/create_sample_tables.sql
--
diff --git a/examples/sample_cube/create_sample_tables.sql 
b/examples/sample_cube/create_sample_tables.sql
index 8399ee8..fa5e25b 100644
--- a/examples/sample_cube/create_sample_tables.sql
+++ b/examples/sample_cube/create_sample_tables.sql
@@ -20,11 +20,11 @@ DROP TABLE IF EXISTS DEFAULT.KYLIN_CAL_DT;
 
 CREATE TABLE DEFAULT.KYLIN_CAL_DT
 (
-CAL_DT date
-,YEAR_BEG_DT date
-,QTR_BEG_DT date
-,MONTH_BEG_DT date
-,WEEK_BEG_DT date
+CAL_DT date COMMENT 'Date, PK'
+,YEAR_BEG_DT date COMMENT 'YEAR Begin Date'
+,QTR_BEG_DT date COMMENT 'Quarter Begin Date'
+,MONTH_BEG_DT date COMMENT 'Month Begin Date'
+,WEEK_BEG_DT date COMMENT 'Week Begin Date'
 ,AGE_FOR_YEAR_ID smallint
 ,AGE_FOR_QTR_ID smallint
 ,AGE_FOR_MONTH_ID smallint
@@ -121,6 +121,7 @@ CAL_DT date
 ,UPD_DATE string
 ,UPD_USER string
 )
+COMMENT 'Date Dimension Table'
 ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
 STORED AS TEXTFILE;
 
@@ -128,28 +129,28 @@ DROP TABLE IF EXISTS DEFAULT.KYLIN_CATEGORY_GROUPINGS;
 
 CREATE TABLE DEFAULT.KYLIN_CATEGORY_GROUPINGS
 (
-LEAF_CATEG_ID bigint
+LEAF_CATEG_ID bigint COMMENT 'Category ID, PK'
 ,LEAF_CATEG_NAME string
-,SITE_ID int
+,SITE_ID int COMMENT 'Site ID, PK'
 ,CATEG_BUSN_MGR string
 ,CATEG_BUSN_UNIT string
 ,REGN_CATEG string
-,USER_DEFINED_FIELD1 string
-,USER_DEFINED_FIELD3 string
+,USER_DEFINED_FIELD1 string COMMENT 'User Defined Field1'
+,USER_DEFINED_FIELD3 string COMMENT 'User Defined Field3'
 ,CRE_DATE string
-,UPD_DATE string
+,UPD_DATE string COMMENT 'Last Updated Date'
 ,CRE_USER string
-,UPD_USER string
+,UPD_USER string COMMENT 'Last Updated User'
 ,META_CATEG_ID decimal
-,META_CATEG_NAME string
+,META_CATEG_NAME string COMMENT 'Level1 Category'
 ,CATEG_LVL2_ID decimal
 ,CATEG_LVL3_ID decimal
 ,CATEG_LVL4_ID decimal
 ,CATEG_LVL5_ID decimal
 ,CATEG_LVL6_ID decimal
 ,CATEG_LVL7_ID decimal
-,CATEG_LVL2_NAME string
-,CATEG_LVL3_NAME string
+,CATEG_LVL2_NAME string 

[16/50] [abbrv] kylin git commit: KYLIN-2083 more RAM estimation test for MeasureAggregator and GTAggregateScanner

[lidong]

KYLIN-2083 more RAM estimation test for MeasureAggregator and GTAggregateScanner


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/460536c0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/460536c0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/460536c0

Branch: refs/heads/master-cdh5.7
Commit: 460536c001575bd09e9f059062efe0d1049d6b31
Parents: ff85103
Author: gaodayue 
Authored: Wed Sep 28 17:09:06 2016 +0800
Committer: gaodayue 
Committed: Tue Oct 11 20:36:11 2016 +0800

--
 core-cube/pom.xml   |   5 +
 .../gridtable/AggregationCacheMemSizeTest.java  | 290 +--
 core-metadata/pom.xml   |   5 +
 .../measure/AggregatorMemEstimateTest.java  | 104 +++
 pom.xml |  35 ++-
 5 files changed, 292 insertions(+), 147 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/460536c0/core-cube/pom.xml
--
diff --git a/core-cube/pom.xml b/core-cube/pom.xml
index 7c9a549..39bba59 100644
--- a/core-cube/pom.xml
+++ b/core-cube/pom.xml
@@ -69,6 +69,11 @@
 junit
 test
 
+
+com.github.jbellis
+jamm
+test
+
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/460536c0/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java
--
diff --git 
a/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java
 
b/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java
index fcd434e..00c0bd0 100644
--- 
a/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java
+++ 
b/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java
@@ -17,197 +17,195 @@
 
 package org.apache.kylin.gridtable;
 
-import java.math.BigDecimal;
-import java.util.Comparator;
-import java.util.Random;
-import java.util.SortedMap;
-import java.util.TreeMap;
-
+import com.google.common.base.Stopwatch;
 import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.measure.MeasureAggregator;
 import org.apache.kylin.measure.basic.BigDecimalSumAggregator;
 import org.apache.kylin.measure.basic.DoubleSumAggregator;
 import org.apache.kylin.measure.basic.LongSumAggregator;
+import org.apache.kylin.measure.bitmap.BitmapAggregator;
+import org.apache.kylin.measure.bitmap.BitmapCounter;
 import org.apache.kylin.measure.hllc.HLLCAggregator;
 import org.apache.kylin.measure.hllc.HyperLogLogPlusCounter;
 import org.apache.kylin.metadata.datatype.DoubleMutable;
 import org.apache.kylin.metadata.datatype.LongMutable;
+import org.github.jamm.MemoryMeter;
 import org.junit.Test;
 
+import java.math.BigDecimal;
+import java.util.*;
+
 public class AggregationCacheMemSizeTest {
+private static final MemoryMeter meter = new MemoryMeter();
+private static final BitmapCounter[] bitmaps = new BitmapCounter[5];
+private static final Random random = new Random();
+
+// consider bitmaps with variant cardinality
+static {
+for (int i = 0; i < bitmaps.length; i++) {
+bitmaps[i] = new BitmapCounter();
+}
 
-public static final int NUM_OF_OBJS = 100 / 2;
+final int totalBits = 1_000_000;
 
-interface CreateAnObject {
-Object create();
-}
+// case 0: sparse, low-cardinality bitmap
+for (int i = 0; i < 100; i++) {
+bitmaps[0].add(random.nextInt(totalBits));
+}
 
-@Test
-public void testHLLCAggregatorSize() throws InterruptedException {
-int est = estimateObjectSize(new CreateAnObject() {
-@Override
-public Object create() {
-HLLCAggregator aggr = new HLLCAggregator(10);
-aggr.aggregate(new HyperLogLogPlusCounter(10));
-return aggr;
+// case 1: 20% full bitmap
+for (int i = 0; i < totalBits; i++) {
+if (random.nextInt(100) < 20) {
+bitmaps[1].add(i);
 }
-});
-System.out.println("HLLC: " + est);
-}
+}
 
-@Test
-public void testBigDecimalAggregatorSize() throws InterruptedException {
-int est = estimateObjectSize(new CreateAnObject() {
-@Override
-public Object create() {
-return newBigDecimalAggr();
+// case 2: half full bitmap
+for (int i = 0; i < totalBits; i++) {
+if (random.nextInt(100) < 50) {
+bitmaps[2].add(i);
 }
+}
 
-});
-

[45/50] [abbrv] kylin git commit: KYLIN-1839 code review

[lidong]

KYLIN-1839 code review


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/aa1550c9
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/aa1550c9
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/aa1550c9

Branch: refs/heads/master-cdh5.7
Commit: aa1550c9ecfb1c047c107fa51187a51e958ab189
Parents: 4473d71
Author: Yang Li 
Authored: Wed Oct 19 08:19:35 2016 +0800
Committer: Yang Li 
Committed: Wed Oct 19 08:19:35 2016 +0800

--
 .../engine/mr/common/AbstractHadoopJob.java | 44 +++-
 1 file changed, 24 insertions(+), 20 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/aa1550c9/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
index bbb1711..f70e3bb 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
@@ -46,7 +46,6 @@ import org.apache.hadoop.conf.Configured;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.protocol.HdfsConstants;
 import org.apache.hadoop.mapreduce.Counters;
 import org.apache.hadoop.mapreduce.InputFormat;
 import org.apache.hadoop.mapreduce.InputSplit;
@@ -80,7 +79,7 @@ public abstract class AbstractHadoopJob extends Configured 
implements Tool {
 protected static final Option OPTION_JOB_NAME = 
OptionBuilder.withArgName(BatchConstants.ARG_JOB_NAME).hasArg().isRequired(true).withDescription("Job
 name. For example, 
Kylin_Cuboid_Builder-clsfd_v2_Step_22-D)").create(BatchConstants.ARG_JOB_NAME);
 protected static final Option OPTION_CUBE_NAME = 
OptionBuilder.withArgName(BatchConstants.ARG_CUBE_NAME).hasArg().isRequired(true).withDescription("Cube
 name. For exmaple, flat_item_cube").create(BatchConstants.ARG_CUBE_NAME);
 protected static final Option OPTION_CUBING_JOB_ID = 
OptionBuilder.withArgName(BatchConstants.ARG_CUBING_JOB_ID).hasArg().isRequired(false).withDescription("ID
 of cubing job executable").create(BatchConstants.ARG_CUBING_JOB_ID);
-//@Deprecated
+//@Deprecated
 protected static final Option OPTION_SEGMENT_NAME = 
OptionBuilder.withArgName(BatchConstants.ARG_SEGMENT_NAME).hasArg().isRequired(true).withDescription("Cube
 segment name").create(BatchConstants.ARG_SEGMENT_NAME);
 protected static final Option OPTION_SEGMENT_ID = 
OptionBuilder.withArgName(BatchConstants.ARG_SEGMENT_ID).hasArg().isRequired(true).withDescription("Cube
 segment id").create(BatchConstants.ARG_SEGMENT_ID);
 protected static final Option OPTION_INPUT_PATH = 
OptionBuilder.withArgName(BatchConstants.ARG_INPUT).hasArg().isRequired(true).withDescription("Input
 path").create(BatchConstants.ARG_INPUT);
@@ -253,10 +252,10 @@ public abstract class AbstractHadoopJob extends 
Configured implements Tool {
 // for KylinJobMRLibDir
 String mrLibDir = kylinConf.getKylinJobMRLibDir();
 if (!StringUtils.isBlank(mrLibDir)) {
-if(kylinDependency.length() > 0) {
-kylinDependency.append(",");
-}
-kylinDependency.append(mrLibDir);
+if (kylinDependency.length() > 0) {
+kylinDependency.append(",");
+}
+kylinDependency.append(mrLibDir);
 }
 
 setJobTmpJarsAndFiles(job, kylinDependency.toString());
@@ -296,7 +295,7 @@ public abstract class AbstractHadoopJob extends Configured 
implements Tool {
 
 try {
 Configuration jobConf = job.getConfiguration();
-FileSystem fs = FileSystem.getLocal(jobConf);
+FileSystem localfs = FileSystem.getLocal(jobConf);
 FileSystem hdfs = FileSystem.get(jobConf);
 
 StringBuilder jarList = new StringBuilder();
@@ -304,20 +303,29 @@ public abstract class AbstractHadoopJob extends 
Configured implements Tool {
 
 for (String fileName : fNameList) {
 Path p = new Path(fileName);
-FileSystem current = 
(fileName.startsWith(HdfsConstants.HDFS_URI_SCHEME) ? hdfs : fs);
-if(!current.exists(p)) {
-logger.warn("The directory '" + fileName + "for kylin 
dependency does not exist!!!");
+if (p.isAbsolute() == false) {
+logger.warn("The directory of kylin dependency '" + 
fileName + "' is not absolute, skip");
+continue;
+

[48/50] [abbrv] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[lidong]

http://git-wip-us.apache.org/repos/asf/kylin/blob/b8391565/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-if (admin.isTableEnabled(table)) {
-admin.disableTable(table);
+if 
(admin.isTableEnabled(TableName.valueOf(table))) {
+admin.disableTable(TableName.valueOf(table));
 }
-admin.deleteTable(table);
+admin.deleteTable(TableName.valueOf(table));
 logger.debug("Dropped htable: " + table);
 output.append("HBase table " + table + " is 
dropped. \n");
 } else {

http://git-wip-us.apache.org/repos/asf/kylin/blob/b8391565/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index a150607..56f867a 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,9 +21,11 @@ package org.apache.kylin.storage.hbase.util;
 import java.io.IOException;
 
 import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.AbstractApplication;
 import org.apache.kylin.common.util.OptionsHelper;
 import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -38,8 +40,8 @@ public class CleanHtableCLI extends AbstractApplication {
 protected static final Logger logger = 
LoggerFactory.getLogger(CleanHtableCLI.class);
 
 private void clean() throws IOException {
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();
 
 for (HTableDescriptor descriptor : hbaseAdmin.listTables()) {
 String name = 

[11/50] [abbrv] kylin git commit: KYLIN-2076 Improve sample cube and data

[lidong]

http://git-wip-us.apache.org/repos/asf/kylin/blob/0c7e2688/examples/sample_cube/data/DEFAULT.KYLIN_SALES.csv
--
diff --git a/examples/sample_cube/data/DEFAULT.KYLIN_SALES.csv 
b/examples/sample_cube/data/DEFAULT.KYLIN_SALES.csv
index 13c7d85..dffa43b 100644
--- a/examples/sample_cube/data/DEFAULT.KYLIN_SALES.csv
+++ b/examples/sample_cube/data/DEFAULT.KYLIN_SALES.csv
@@ -1,1 +1,1 @@
-0,2012-12-14,Others,88750,0,11,36.2828,0,1349
-0,2012-08-28,Others,175750,0,13,23.8563,0,1927
-0,2012-02-16,ABIN,148324,15,13,88.3418,0,1005
-0,2013-10-19,FP-non GTC,37831,0,13,47.3015,0,1209
-0,2012-10-22,Others,140746,100,11,83.4540,0,1154
-0,2013-01-25,FP-GTC,16509,0,-99,22.9896,0,1372
-0,2013-04-04,Others,963,0,13,88.5907,0,1648
-0,2012-04-11,Others,15687,0,15,88.1940,0,1866
-0,2013-01-19,ABIN,60606,3,13,77.9727,0,1936
-0,2012-04-30,FP-non GTC,106246,0,14,52.2950,0,1047
-0,2013-02-03,Auction,45333,0,16,56.3584,0,1470
-0,2012-09-02,FP-GTC,25147,0,-99,15.3553,0,1203
-0,2013-07-30,Others,24760,0,16,25.0770,0,1382
-0,2012-04-17,Auction,31519,0,16,72.7384,0,1069
-0,2013-04-30,FP-GTC,51582,0,15,75.8200,0,1085
-0,2013-03-23,FP-GTC,16509,0,15,8.6653,0,1559
-0,2013-04-01,ABIN,175750,3,16,64.2802,0,1507
-0,2013-07-12,ABIN,50677,0,-99,24.5987,0,1938
-0,2012-04-23,FP-GTC,45333,0,5,72.6553,0,1863
-0,2012-12-19,ABIN,3838,0,12,87.4142,0,1612
-0,2012-08-15,FP-GTC,62179,0,16,67.4238,0,1839
-0,2013-08-22,Auction,1357,0,5,18.3600,0,1303
-0,2013-08-23,ABIN,31519,0,5,67.6344,0,1877
-0,2013-09-12,FP-GTC,223,0,14,71.7898,0,1848
-0,2012-03-14,FP-GTC,132939,0,15,26.1496,0,1815
-0,2013-07-13,Others,67703,3,16,83.9661,0,1858
-0,2013-12-15,FP-non GTC,100847,0,5,34.3451,0,1387
-0,2012-08-30,Auction,62179,0,11,21.0496,0,1393
-0,2013-01-10,FP-GTC,48027,0,13,23.4142,0,1757
-0,2013-01-25,ABIN,145970,0,5,26.7842,0,1402
-0,2012-07-05,Auction,48027,0,-99,46.7320,0,1972
-0,2012-11-07,FP-GTC,20213,0,-99,19.7270,0,1873
-0,2013-06-20,ABIN,100847,0,5,76.2783,0,1409
-0,2012-10-08,ABIN,26262,0,5,48.7447,0,1029
-0,2013-04-30,ABIN,87118,0,15,63.2187,0,1529
-0,2013-04-24,FP-non GTC,20213,0,16,39.5057,0,1329
-0,2012-05-13,Others,164261,0,13,84.1246,0,1688
-0,2013-08-02,ABIN,31519,0,5,37.1504,0,1905
-0,2012-04-03,Auction,164261,0,11,92.0974,0,1341
-0,2012-09-02,FP-GTC,31519,0,14,87.7829,0,1460
-0,2013-10-05,ABIN,13987,0,16,74.0719,0,1381
-0,2012-01-13,FP-GTC,48027,0,15,71.2637,0,1915
-0,2013-03-01,Others,13836,0,14,16.7288,0,1953
-0,2012-09-12,FP-non GTC,16509,0,5,12.2933,0,1277
-0,2012-02-07,Auction,45333,0,5,64.9770,0,1040
-0,2013-12-14,FP-non GTC,158798,0,16,72.4413,0,1500
-0,2013-10-13,Auction,31519,0,5,79.3053,0,1816
-0,2012-06-11,ABIN,4943,0,5,11.6942,0,1696
-0,2012-10-18,ABIN,80053,0,-99,54.0933,0,1604
-0,2012-11-03,Auction,20886,0,5,9.8258,0,1764
-0,2012-01-15,Auction,44079,0,14,13.0371,0,1343
-0,2012-07-03,FP-GTC,159184,0,-99,92.5314,0,1724
-0,2012-02-16,FP-GTC,314,0,13,49.1825,0,1334
-0,2012-06-17,FP-non GTC,31387,3,12,0.6677,0,1603
-0,2012-07-15,FP-non GTC,32876,0,-99,50.9634,0,1248
-0,2012-07-24,FP-GTC,150265,15,16,57.5645,0,1653
-0,2012-10-06,Auction,145970,0,5,44.3091,0,1082
-0,2013-06-14,ABIN,80053,0,5,20.2388,0,1632
-0,2013-10-10,FP-non GTC,15868,0,11,28.6290,0,1523
-0,2012-09-30,ABIN,31673,0,12,75.8658,0,1719
-0,2012-04-02,FP-GTC,63861,0,5,25.8748,0,1311
-0,2012-03-16,Others,314,0,-99,9.2733,0,1409
-0,2012-09-30,FP-non GTC,60340,0,14,74.6610,0,1790
-0,2012-03-20,Others,87118,0,5,79.3781,0,1384
-0,2012-09-12,FP-GTC,1120,3,5,61.4408,0,1645
-0,2013-12-10,FP-non GTC,20886,0,11,14.8140,0,1943
-0,2012-12-05,FP-GTC,106246,0,12,24.9380,0,1858
-0,2012-08-21,Auction,43479,0,12,22.2008,0,1310
-0,2012-11-30,FP-GTC,175750,0,15,35.1161,0,1440
-0,2013-10-10,Others,95672,0,11,82.3718,0,1128
-0,2012-10-17,Auction,80053,0,15,9.8166,0,1105
-0,2012-03-16,FP-non GTC,57784,0,5,66.2369,0,1439
-0,2012-11-14,ABIN,65,0,11,17.5513,0,1855
-0,2013-06-24,FP-GTC,16509,0,15,53.9195,0,1130
-0,2012-08-13,FP-non GTC,65,0,5,69.8447,0,1548
-0,2012-01-03,Others,38238,0,11,90.3722,0,1739
-0,2013-11-24,FP-non GTC,175750,3,13,47.3441,0,1291
-0,2013-11-08,Auction,106246,0,5,38.6729,0,1310
-0,2013-04-07,Others,73506,0,-99,69.8999,0,1235
-0,2012-10-15,FP-GTC,314,0,13,44.1598,0,1265
-0,2012-01-20,FP-GTC,80287,0,-99,85.7263,0,1704
-0,2012-08-17,ABIN,32876,0,16,21.2934,0,1163
-0,2013-07-09,Auction,37831,0,15,38.2213,0,1557
-0,2013-09-21,Others,45333,0,13,25.5384,0,1184
-0,2013-10-28,FP-non GTC,11554,0,16,89.9654,0,1940
-0,2013-04-05,Auction,11554,0,14,41.8676,0,1384
-0,2013-03-25,FP-GTC,43398,0,14,75.0110,0,1697
-0,2012-05-03,ABIN,33038,15,12,64.6735,0,1147

[22/50] [abbrv] kylin git commit: KYLIN-1967 Skip empty cube segment

[lidong]

KYLIN-1967 Skip empty cube segment


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e6698c8f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e6698c8f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e6698c8f

Branch: refs/heads/master-cdh5.7
Commit: e6698c8f054ce925adb041a85c21af8e886aa768
Parents: 5974fc0
Author: Li Yang 
Authored: Thu Oct 13 18:55:11 2016 +0800
Committer: Li Yang 
Committed: Thu Oct 13 18:55:59 2016 +0800

--
 .../kylin/storage/gtrecord/CubeSegmentScanner.java   |  2 ++
 .../kylin/storage/gtrecord/GTCubeStorageQueryBase.java   | 11 ++-
 2 files changed, 12 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/e6698c8f/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
--
diff --git 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
index 926ba48..6e19c09 100644
--- 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
+++ 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
@@ -69,6 +69,8 @@ public class CubeSegmentScanner implements IGTScanner {
 CubeScanRangePlanner scanRangePlanner;
 try {
 scanRangePlanner = new CubeScanRangePlanner(cubeSeg, cuboid, 
filter, dimensions, groups, metrics, context);
+} catch (RuntimeException e) {
+throw e;
 } catch (Exception e) {
 throw new RuntimeException(e);
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/e6698c8f/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
--
diff --git 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
index 4ba134d..fa23be1 100644
--- 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
+++ 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
@@ -133,7 +133,16 @@ public abstract class GTCubeStorageQueryBase implements 
IStorageQuery {
 continue;
 }
 }
-scanner = new CubeSegmentScanner(cubeSeg, cuboid, dimensionsD, 
groupsD, metrics, filterD, context, getGTStorage());
+try {
+scanner = new CubeSegmentScanner(cubeSeg, cuboid, dimensionsD, 
groupsD, metrics, filterD, context, getGTStorage());
+} catch (IllegalArgumentException ex) {
+// ref KYLIN-1967, real empty segment can trigger dictionary 
exception -- IllegalArgumentException: Value not exists!
+if (cubeSeg.getInputRecords() == 0) {
+logger.warn("cube segment {} input record is 0, skip it 
still", cubeSeg);
+continue;
+}
+throw ex;
+}
 scanners.add(scanner);
 }
 

[15/50] [abbrv] kylin git commit: allow cleanupOldStorage() be overwritted

[lidong]

allow cleanupOldStorage() be overwritted


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4be729e0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4be729e0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4be729e0

Branch: refs/heads/master-hbase1.x
Commit: 4be729e0f7616b24cb289ead635b882abc515e1b
Parents: fad9e81
Author: shaofengshi 
Authored: Tue Oct 11 17:21:08 2016 +0800
Committer: shaofengshi 
Committed: Tue Oct 11 17:21:08 2016 +0800

--
 .../java/org/apache/kylin/provision/BuildCubeWithStream.java   | 6 +-
 1 file changed, 1 insertion(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/4be729e0/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java 
b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
index d2e46a5..2faa8d0 100644
--- a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
+++ b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
@@ -301,7 +301,7 @@ public class BuildCubeWithStream {
 HBaseMetadataTestCase.staticCleanupTestMetadata();
 }
 
-private static void cleanupOldStorage() throws Exception {
+protected void cleanupOldStorage() throws Exception {
 String[] args = { "--delete", "true" };
 StorageCleanupJob cli = new StorageCleanupJob();
 cli.execute(args);
@@ -321,10 +321,6 @@ public class BuildCubeWithStream {
 } catch (Throwable e) {
 logger.error("error", e);
 System.exit(1);
-} finally {
-if (buildCubeWithStream != null) {
-buildCubeWithStream.after();
-}
 }
 
 }

[13/50] [abbrv] kylin git commit: KYLIN-2076 Improve sample cube and data

[lidong]

KYLIN-2076 Improve sample cube and data

Signed-off-by: Li Yang 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0c7e2688
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0c7e2688
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0c7e2688

Branch: refs/heads/master-hbase1.x
Commit: 0c7e2688e80822cfbdad901cef909884bfa79ee8
Parents: 726b0ec
Author: Cheng Wang 
Authored: Mon Oct 10 17:49:33 2016 +0800
Committer: Li Yang 
Committed: Mon Oct 10 18:02:04 2016 +0800

--
 examples/sample_cube/create_sample_tables.sql   | 2 +
 .../sample_cube/data/DEFAULT.KYLIN_SALES.csv| 2 -
 .../cube_desc/kylin_sales_cube_desc.json|18 +-
 .../template/model_desc/kylin_sales_model.json  | 4 +-
 .../template/table/DEFAULT.KYLIN_SALES.json |10 +-
 5 files changed, 10030 insertions(+), 10004 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/0c7e2688/examples/sample_cube/create_sample_tables.sql
--
diff --git a/examples/sample_cube/create_sample_tables.sql 
b/examples/sample_cube/create_sample_tables.sql
index 5ed7b9b..8399ee8 100644
--- a/examples/sample_cube/create_sample_tables.sql
+++ b/examples/sample_cube/create_sample_tables.sql
@@ -181,6 +181,8 @@ TRANS_ID bigint
 ,PRICE decimal(19,4)
 ,ITEM_COUNT bigint
 ,SELLER_ID bigint
+,USER_ID string
+,REGION string
 )
 ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
 STORED AS TEXTFILE;

[35/50] [abbrv] kylin git commit: KYLIN-2098 TopN support query UHC column without sorting by sum value

[lidong]

KYLIN-2098 TopN support query UHC column without sorting by sum value

Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3924672b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3924672b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3924672b

Branch: refs/heads/master-hbase1.x
Commit: 3924672be05305a9374b4ba757500d89af8e125c
Parents: ef0c9e5
Author: shaofengshi 
Authored: Mon Oct 17 13:51:05 2016 +0800
Committer: shaofengshi 
Committed: Mon Oct 17 13:51:53 2016 +0800

--
 .../kylin/measure/topn/TopNMeasureType.java | 55 +---
 1 file changed, 35 insertions(+), 20 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/3924672b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
 
b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
index b67c374..fcf6d5e 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
@@ -231,34 +231,49 @@ public class TopNMeasureType extends 
MeasureType {
 // TopN measure can (and only can) provide one numeric measure and one 
literal dimension
 // e.g. select seller, sum(gmv) from ... group by seller order by 2 
desc limit 100
 
-// check digest requires only one measure
-if (digest.aggregations.size() != 1)
-return null;
-
-// the measure function must be SUM
-FunctionDesc onlyFunction = digest.aggregations.iterator().next();
-if (isTopNCompatibleSum(topN.getFunction(), onlyFunction) == false)
-return null;
-
 List literalCol = getTopNLiteralColumn(topN.getFunction());
+for (TblColRef colRef : literalCol) {
+if (digest.filterColumns.contains(colRef) == true) {
+// doesn't allow filtering by topn literal column
+return null;
+}
+}
+
 if (unmatchedDimensions.containsAll(literalCol) == false)
 return null;
 if (digest.groupbyColumns.containsAll(literalCol) == false)
 return null;
 
-for (TblColRef colRef : literalCol) {
-if (digest.filterColumns.contains(colRef) == true) {
+// check digest requires only one measure
+if (digest.aggregations.size() == 1) {
+
+// the measure function must be SUM
+FunctionDesc onlyFunction = digest.aggregations.iterator().next();
+if (isTopNCompatibleSum(topN.getFunction(), onlyFunction) == false)
 return null;
-}
+
+unmatchedDimensions.removeAll(literalCol);
+unmatchedAggregations.remove(onlyFunction);
+return new CapabilityInfluence() {
+@Override
+public double suggestCostMultiplier() {
+return 0.3; // make sure TopN get ahead of other matched 
realizations
+}
+};
 }
-unmatchedDimensions.removeAll(literalCol);
-unmatchedAggregations.remove(onlyFunction);
-return new CapabilityInfluence() {
-@Override
-public double suggestCostMultiplier() {
-return 0.3; // make sure TopN get ahead of other matched 
realizations
-}
-};
+
+if (digest.aggregations.size() == 0 ) {
+// directly query the UHC column without sorting
+unmatchedDimensions.removeAll(literalCol);
+return new CapabilityInfluence() {
+@Override
+public double suggestCostMultiplier() {
+return 2.0; // topn can answer but with a higher cost
+}
+};
+}
+
+return null;
 }
 
 private boolean isTopNCompatibleSum(FunctionDesc topN, FunctionDesc sum) {

[30/50] [abbrv] kylin git commit: The SweetAlert at the front page may out of the page if the content is too long

[lidong]

The SweetAlert at the front page may out of the page if the content is too long

Signed-off-by: Jason 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/70c85a7a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/70c85a7a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/70c85a7a

Branch: refs/heads/master-hbase1.x
Commit: 70c85a7a1279ed0246dbc3c61cc47275cd1b31c5
Parents: 81403c1
Author: liguohui 
Authored: Thu Oct 13 15:05:03 2016 +0800
Committer: Jason 
Committed: Mon Oct 17 11:20:01 2016 +0800

--
 webapp/app/less/component.less | 2 ++
 1 file changed, 2 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/70c85a7a/webapp/app/less/component.less
--
diff --git a/webapp/app/less/component.less b/webapp/app/less/component.less
index e23859d..ed1c9b7 100644
--- a/webapp/app/less/component.less
+++ b/webapp/app/less/component.less
@@ -1169,6 +1169,8 @@ ul.abn-tree li.abn-tree-row a {
 
 .sweet-alert .lead.text-muted {
   word-break: break-all;
+  max-height: 300px;
+  overflow: auto;
 }
 
 .modal-body.streaming-source .ace_editor {

[41/50] [abbrv] kylin git commit: KYLIN-2091 bug fix

[lidong]

KYLIN-2091 bug fix

Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c13ab1c3
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c13ab1c3
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c13ab1c3

Branch: refs/heads/master-hbase1.x
Commit: c13ab1c324bf6a56e75365e0e1ab5ecd48fd048a
Parents: 07e81fd
Author: shaofengshi 
Authored: Tue Oct 18 10:43:13 2016 +0800
Committer: shaofengshi 
Committed: Tue Oct 18 10:43:13 2016 +0800

--
 .../main/java/org/apache/kylin/cube/CubeSegment.java| 12 
 .../apache/kylin/source/kafka/job/SeekOffsetStep.java   |  2 ++
 2 files changed, 2 insertions(+), 12 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/c13ab1c3/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
index eb5b389..fdf1fb0 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
@@ -564,12 +564,6 @@ public class CubeSegment implements 
Comparable, IBuildable, ISegmen
 
 public void setSourcePartitionOffsetEnd(Map 
sourcePartitionOffsetEnd) {
 this.sourcePartitionOffsetEnd = sourcePartitionOffsetEnd;
-long total = 0;
-for (Long x : sourcePartitionOffsetEnd.values()) {
-total += x;
-}
-
-this.sourceOffsetEnd = total;
 }
 
 public Map getSourcePartitionOffsetStart() {
@@ -578,11 +572,5 @@ public class CubeSegment implements 
Comparable, IBuildable, ISegmen
 
 public void setSourcePartitionOffsetStart(Map 
sourcePartitionOffsetStart) {
 this.sourcePartitionOffsetStart = sourcePartitionOffsetStart;
-long total = 0;
-for (Long x : sourcePartitionOffsetStart.values()) {
-total += x;
-}
-
-this.sourceOffsetStart = total;
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/c13ab1c3/source-kafka/src/main/java/org/apache/kylin/source/kafka/job/SeekOffsetStep.java
--
diff --git 
a/source-kafka/src/main/java/org/apache/kylin/source/kafka/job/SeekOffsetStep.java
 
b/source-kafka/src/main/java/org/apache/kylin/source/kafka/job/SeekOffsetStep.java
index 2a3dbb5..a26f39d 100644
--- 
a/source-kafka/src/main/java/org/apache/kylin/source/kafka/job/SeekOffsetStep.java
+++ 
b/source-kafka/src/main/java/org/apache/kylin/source/kafka/job/SeekOffsetStep.java
@@ -116,6 +116,8 @@ public class SeekOffsetStep extends AbstractExecutable {
 }
 
 if (totalEndOffset > totalStartOffset) {
+segment.setSourceOffsetStart(totalStartOffset);
+segment.setSourceOffsetEnd(totalEndOffset);
 segment.setSourcePartitionOffsetStart(startOffsets);
 segment.setSourcePartitionOffsetEnd(endOffsets);
 segment.setName(CubeSegment.makeSegmentName(0, 0, 
totalStartOffset, totalEndOffset));

[19/50] [abbrv] kylin git commit: KYLIN-2085 Consider dynamicVariables in CompareTupleFilter operator swap

[lidong]

KYLIN-2085 Consider dynamicVariables in CompareTupleFilter operator swap


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8fbfc580
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8fbfc580
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8fbfc580

Branch: refs/heads/master-hbase1.x
Commit: 8fbfc580c1b89e3e0065027eb22f8e3e5b7e0c67
Parents: 58afe01
Author: lidongsjtu 
Authored: Wed Oct 12 13:44:11 2016 +0800
Committer: lidongsjtu 
Committed: Wed Oct 12 16:32:44 2016 +0800

--
 .../java/org/apache/kylin/metadata/filter/CompareTupleFilter.java | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/8fbfc580/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
index fe51710..c7a3721 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
@@ -75,7 +75,8 @@ public class CompareTupleFilter extends TupleFilter {
 }
 this.column = columnFilter.getColumn();
 // if value is before column, we need to reverse the operator. 
e.g. "1 >= c1" => "c1 <= 1"
-if (!this.conditionValues.isEmpty() && needSwapOperator()) {
+// children.size() > 1 means already added one conditionValue or 
dynamicVariable
+if (this.children.size() > 1 && needSwapOperator()) {
 this.operator = SWAP_OP_MAP.get(this.operator);
 TupleFilter last = this.children.remove(this.children.size() - 
1);
 this.children.add(0, last);

[11/50] [abbrv] kylin git commit: KYLIN-2076 Improve sample cube and data

[lidong]

http://git-wip-us.apache.org/repos/asf/kylin/blob/0c7e2688/examples/sample_cube/template/cube_desc/kylin_sales_cube_desc.json
--
diff --git a/examples/sample_cube/template/cube_desc/kylin_sales_cube_desc.json 
b/examples/sample_cube/template/cube_desc/kylin_sales_cube_desc.json
index 99f8f46..cc1cd76 100644
--- a/examples/sample_cube/template/cube_desc/kylin_sales_cube_desc.json
+++ b/examples/sample_cube/template/cube_desc/kylin_sales_cube_desc.json
@@ -35,6 +35,16 @@
 "table" : "DEFAULT.KYLIN_SALES",
 "column" : "LSTG_FORMAT_NAME",
 "derived" : null
+  }, {
+"name" : "USER_ID",
+"table" : "DEFAULT.KYLIN_SALES",
+"column" : "USER_ID",
+"derived" : null
+  }, {
+"name" : "REGION",
+"table" : "DEFAULT.KYLIN_SALES",
+"column" : "REGION",
+"derived" : null
   } ],
   "measures" : [ {
 "name" : "GMV_SUM",
@@ -145,6 +155,12 @@
   "column" : "LSTG_FORMAT_NAME",
   "encoding" : "fixed_length:12"
 }, {
+  "column" : "USER_ID",
+  "encoding" : "dict"
+}, {
+  "column" : "REGION",
+  "encoding" : "dict"
+}, {
   "column" : "LSTG_SITE_ID",
   "encoding" : "dict"
 } ]
@@ -180,4 +196,4 @@
   "engine_type" : %default_engine_type%,
   "storage_type" : %default_storage_type%,
   "override_kylin_properties" : { }
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/0c7e2688/examples/sample_cube/template/model_desc/kylin_sales_model.json
--
diff --git a/examples/sample_cube/template/model_desc/kylin_sales_model.json 
b/examples/sample_cube/template/model_desc/kylin_sales_model.json
index 7ae47c1..13fc356 100644
--- a/examples/sample_cube/template/model_desc/kylin_sales_model.json
+++ b/examples/sample_cube/template/model_desc/kylin_sales_model.json
@@ -24,7 +24,7 @@
 "columns" : [ "USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3", "UPD_DATE", 
"UPD_USER", "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME" ]
   }, {
 "table" : "DEFAULT.KYLIN_SALES",
-"columns" : [ "LSTG_FORMAT_NAME", "SELLER_ID", "PART_DT" ]
+"columns" : [ "LSTG_FORMAT_NAME", "SELLER_ID", "PART_DT", "USER_ID", 
"REGION" ]
   } ],
   "metrics": [
 "PRICE",
@@ -40,4 +40,4 @@
 "partition_date_start" : 132537600,
 "partition_type" : "APPEND"
   }
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/0c7e2688/examples/sample_cube/template/table/DEFAULT.KYLIN_SALES.json
--
diff --git a/examples/sample_cube/template/table/DEFAULT.KYLIN_SALES.json 
b/examples/sample_cube/template/table/DEFAULT.KYLIN_SALES.json
index 4927343..2bccbeb 100644
--- a/examples/sample_cube/template/table/DEFAULT.KYLIN_SALES.json
+++ b/examples/sample_cube/template/table/DEFAULT.KYLIN_SALES.json
@@ -38,7 +38,15 @@
 "id" : "9",
 "name" : "SELLER_ID",
 "datatype" : "bigint"
+  }, {
+"id" : "10",
+"name" : "USER_ID",
+"datatype" : "string"
+  }, {
+"id" : "11",
+"name" : "REGION",
+"datatype" : "string"
   } ],
   "database" : "DEFAULT",
   "last_modified" : 0
-}
\ No newline at end of file
+}

[23/50] [abbrv] kylin git commit: KYLIN-1967 Skip empty cube segment

[lidong]

KYLIN-1967 Skip empty cube segment


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e6698c8f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e6698c8f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e6698c8f

Branch: refs/heads/master-hbase1.x
Commit: e6698c8f054ce925adb041a85c21af8e886aa768
Parents: 5974fc0
Author: Li Yang 
Authored: Thu Oct 13 18:55:11 2016 +0800
Committer: Li Yang 
Committed: Thu Oct 13 18:55:59 2016 +0800

--
 .../kylin/storage/gtrecord/CubeSegmentScanner.java   |  2 ++
 .../kylin/storage/gtrecord/GTCubeStorageQueryBase.java   | 11 ++-
 2 files changed, 12 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/e6698c8f/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
--
diff --git 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
index 926ba48..6e19c09 100644
--- 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
+++ 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
@@ -69,6 +69,8 @@ public class CubeSegmentScanner implements IGTScanner {
 CubeScanRangePlanner scanRangePlanner;
 try {
 scanRangePlanner = new CubeScanRangePlanner(cubeSeg, cuboid, 
filter, dimensions, groups, metrics, context);
+} catch (RuntimeException e) {
+throw e;
 } catch (Exception e) {
 throw new RuntimeException(e);
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/e6698c8f/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
--
diff --git 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
index 4ba134d..fa23be1 100644
--- 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
+++ 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
@@ -133,7 +133,16 @@ public abstract class GTCubeStorageQueryBase implements 
IStorageQuery {
 continue;
 }
 }
-scanner = new CubeSegmentScanner(cubeSeg, cuboid, dimensionsD, 
groupsD, metrics, filterD, context, getGTStorage());
+try {
+scanner = new CubeSegmentScanner(cubeSeg, cuboid, dimensionsD, 
groupsD, metrics, filterD, context, getGTStorage());
+} catch (IllegalArgumentException ex) {
+// ref KYLIN-1967, real empty segment can trigger dictionary 
exception -- IllegalArgumentException: Value not exists!
+if (cubeSeg.getInputRecords() == 0) {
+logger.warn("cube segment {} input record is 0, skip it 
still", cubeSeg);
+continue;
+}
+throw ex;
+}
 scanners.add(scanner);
 }
 

[33/50] [abbrv] kylin git commit: minor, mv method from query controller to service

[lidong]

minor, mv method from query controller to service

Signed-off-by: Li Yang 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/44cf9fba
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/44cf9fba
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/44cf9fba

Branch: refs/heads/master-hbase1.x
Commit: 44cf9fba5900252fb3b4364d4bd129d83212d51c
Parents: ef44b7f
Author: Roger Shi 
Authored: Mon Oct 17 13:21:37 2016 +0800
Committer: Li Yang 
Committed: Mon Oct 17 13:33:08 2016 +0800

--
 .../kylin/rest/controller/QueryController.java  | 136 +--
 .../apache/kylin/rest/service/CacheService.java |   5 +-
 .../apache/kylin/rest/service/QueryService.java | 132 +-
 .../rest/controller/QueryControllerTest.java|   2 +-
 4 files changed, 134 insertions(+), 141 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/44cf9fba/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java
 
b/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java
index 9471937..c5f896d 100644
--- 
a/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java
+++ 
b/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java
@@ -23,15 +23,10 @@ import java.sql.SQLException;
 import java.util.ArrayList;
 import java.util.List;
 
-import javax.annotation.PostConstruct;
 import javax.servlet.http.HttpServletResponse;
 
 import org.apache.commons.io.IOUtils;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.debug.BackdoorToggles;
-import org.apache.kylin.rest.constant.Constant;
 import org.apache.kylin.rest.exception.InternalErrorException;
-import org.apache.kylin.rest.metrics.QueryMetricsFacade;
 import org.apache.kylin.rest.model.Query;
 import org.apache.kylin.rest.model.SelectedColumnMeta;
 import org.apache.kylin.rest.model.TableMeta;
@@ -41,12 +36,9 @@ import org.apache.kylin.rest.request.SQLRequest;
 import org.apache.kylin.rest.request.SaveSqlRequest;
 import org.apache.kylin.rest.response.SQLResponse;
 import org.apache.kylin.rest.service.QueryService;
-import org.apache.kylin.rest.util.QueryUtil;
-import org.apache.kylin.storage.exception.ScanOutOfLimitException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.security.access.AccessDeniedException;
 import org.springframework.security.core.context.SecurityContextHolder;
 import org.springframework.stereotype.Controller;
 import org.springframework.web.bind.annotation.PathVariable;
@@ -58,12 +50,6 @@ import org.supercsv.io.CsvListWriter;
 import org.supercsv.io.ICsvListWriter;
 import org.supercsv.prefs.CsvPreference;
 
-import com.google.common.base.Preconditions;
-
-import net.sf.ehcache.Cache;
-import net.sf.ehcache.CacheManager;
-import net.sf.ehcache.Element;
-
 /**
  * Handle query requests.
  * 
@@ -74,31 +60,20 @@ public class QueryController extends BasicController {
 
 private static final Logger logger = 
LoggerFactory.getLogger(QueryController.class);
 
-public static final String SUCCESS_QUERY_CACHE = "StorageCache";
-public static final String EXCEPTION_QUERY_CACHE = "ExceptionQueryCache";
-
 @Autowired
 private QueryService queryService;
 
-@Autowired
-private CacheManager cacheManager;
-
-@PostConstruct
-public void init() throws IOException {
-Preconditions.checkNotNull(cacheManager, "cacheManager is not injected 
yet");
-}
-
 @RequestMapping(value = "/query", method = RequestMethod.POST)
 @ResponseBody
 public SQLResponse query(@RequestBody SQLRequest sqlRequest) {
-return doQueryWithCache(sqlRequest);
+return queryService.doQueryWithCache(sqlRequest);
 }
 
 // TODO should be just "prepare" a statement, get back expected 
ResultSetMetaData
 @RequestMapping(value = "/query/prestate", method = RequestMethod.POST, 
produces = "application/json")
 @ResponseBody
 public SQLResponse prepareQuery(@RequestBody PrepareSqlRequest sqlRequest) 
{
-return doQueryWithCache(sqlRequest);
+return queryService.doQueryWithCache(sqlRequest);
 }
 
 @RequestMapping(value = "/saved_queries", method = RequestMethod.POST)
@@ -127,7 +102,7 @@ public class QueryController extends BasicController {
 @RequestMapping(value = "/query/format/{format}", method = 
RequestMethod.GET)
 @ResponseBody
 public void downloadQueryResult(@PathVariable String format, SQLRequest 
sqlRequest, 

[38/50] [abbrv] kylin git commit: delete the blank line code

[lidong]

delete the blank line code

Signed-off-by: shaofengshi 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/cc2b59fa
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/cc2b59fa
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/cc2b59fa

Branch: refs/heads/master-hbase1.x
Commit: cc2b59faf1a12e76e6af83ba3a19fbe27ba54cc3
Parents: ac356f0
Author: lijieliang 
Authored: Fri Oct 14 14:28:01 2016 +0800
Committer: shaofengshi 
Committed: Mon Oct 17 18:25:26 2016 +0800

--
 .../src/main/java/org/apache/kylin/source/hive/HiveMRInput.java | 1 -
 1 file changed, 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/cc2b59fa/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
--
diff --git 
a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java 
b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
index 2ec1fbb..4ec8d3d 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
@@ -130,7 +130,6 @@ public class HiveMRInput implements IMRInput {
 @Override
 public void addStepPhase1_CreateFlatTable(DefaultChainedExecutable 
jobFlow) {
 final String cubeName = 
CubingExecutableUtil.getCubeName(jobFlow.getParams());
-
 final KylinConfig kylinConfig = 
CubeManager.getInstance(conf.getConfig()).getCube(cubeName).getConfig();
 
 String createFlatTableMethod = 
kylinConfig.getCreateFlatHiveTableMethod();

[37/50] [abbrv] kylin git commit: KYLIN-2091 Add API to init the start-point (of each parition) for streaming cube

[lidong]

KYLIN-2091 Add API to init the start-point (of each parition) for streaming cube

Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c92f79ad
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c92f79ad
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c92f79ad

Branch: refs/heads/master-hbase1.x
Commit: c92f79ad39e562ed32fff1a55eb979f0593ed6e3
Parents: 5429006
Author: shaofengshi 
Authored: Mon Oct 17 17:40:57 2016 +0800
Committer: shaofengshi 
Committed: Mon Oct 17 17:53:56 2016 +0800

--
 .../apache/kylin/common/KylinConfigBase.java|  2 +-
 .../java/org/apache/kylin/cube/CubeSegment.java | 12 ++
 .../org/apache/kylin/cube/model/CubeDesc.java   | 13 ++
 .../kylin/measure/topn/TopNMeasureType.java |  2 +-
 server-base/pom.xml |  7 
 .../kylin/rest/controller/CubeController.java   | 10 ++---
 .../rest/controller/CubeDescController.java | 44 +++-
 .../kylin/source/kafka/job/SeekOffsetStep.java  |  8 +++-
 .../kylin/source/kafka/util/KafkaClient.java| 22 +-
 9 files changed, 108 insertions(+), 12 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/c92f79ad/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 4942081..73ac788 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -804,7 +804,7 @@ abstract public class KylinConfigBase implements 
Serializable {
 }
 
 public int getMaxBuildingSegments() {
-return Integer.parseInt(getOptional("kylin.cube.building.segment.max", 
"2"));
+return Integer.parseInt(getOptional("kylin.cube.building.segment.max", 
"10"));
 }
 
 public void setMaxBuildingSegments(int maxBuildingSegments) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/c92f79ad/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
index fdf1fb0..eb5b389 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
@@ -564,6 +564,12 @@ public class CubeSegment implements 
Comparable, IBuildable, ISegmen
 
 public void setSourcePartitionOffsetEnd(Map 
sourcePartitionOffsetEnd) {
 this.sourcePartitionOffsetEnd = sourcePartitionOffsetEnd;
+long total = 0;
+for (Long x : sourcePartitionOffsetEnd.values()) {
+total += x;
+}
+
+this.sourceOffsetEnd = total;
 }
 
 public Map getSourcePartitionOffsetStart() {
@@ -572,5 +578,11 @@ public class CubeSegment implements 
Comparable, IBuildable, ISegmen
 
 public void setSourcePartitionOffsetStart(Map 
sourcePartitionOffsetStart) {
 this.sourcePartitionOffsetStart = sourcePartitionOffsetStart;
+long total = 0;
+for (Long x : sourcePartitionOffsetStart.values()) {
+total += x;
+}
+
+this.sourceOffsetStart = total;
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/c92f79ad/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java 
b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index 4195451..bf3724a 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -170,6 +170,10 @@ public class CubeDesc extends RootPersistentEntity 
implements IEngineAware {
 
 private Map extendedColumnToHosts = 
Maps.newHashMap();
 
+@JsonProperty("partition_offset_start")
+@JsonInclude(JsonInclude.Include.NON_EMPTY)
+private Map partitionOffsetStart = Maps.newHashMap();
+
 public boolean isEnableSharding() {
 //in the future may extend to other storage that is shard-able
 return storageType != IStorageAware.ID_HBASE && storageType != 
IStorageAware.ID_HYBRID;
@@ -1011,6 +1015,14 @@ public class CubeDesc extends RootPersistentEntity 
implements IEngineAware {
 this.partitionDateEnd = partitionDateEnd;
 }
 
+public 

[48/50] [abbrv] kylin git commit: KYLIN-2105 add QueryId

[lidong]

KYLIN-2105 add QueryId


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/db09f5f9
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/db09f5f9
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/db09f5f9

Branch: refs/heads/master-hbase1.x
Commit: db09f5f9cae5a3d3ff731221cbb1c026da4f4e41
Parents: 6cafc42
Author: gaodayue 
Authored: Wed Oct 19 08:36:53 2016 +0800
Committer: gaodayue 
Committed: Wed Oct 19 08:36:53 2016 +0800

--
 .../kylin/common/debug/BackdoorToggles.java |   6 +
 .../apache/kylin/common/util/SetThreadName.java |  49 +
 .../apache/kylin/rest/service/QueryService.java |  32 ++-
 .../kylin/rest/util/QueryIdGenerator.java   |  45 +
 .../kylin/rest/util/QueryIdGeneratorTest.java   | 106 ++
 .../src/main/resources/applicationContext.xml   |   1 +
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java |   7 +-
 .../coprocessor/endpoint/CubeVisitService.java  |   4 +-
 .../endpoint/generated/CubeVisitProtos.java | 197 +--
 .../endpoint/protobuf/CubeVisit.proto   |   1 +
 10 files changed, 420 insertions(+), 28 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/db09f5f9/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java 
b/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
index f88781a..f7c90aa 100644
--- 
a/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
+++ 
b/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
@@ -65,6 +65,10 @@ public class BackdoorToggles {
 return Integer.valueOf(v);
 }
 
+public static String getQueryId() {
+return getString(KEY_QUERY_ID);
+}
+
 public static Pair getShardAssignment() {
 String v = getString(DEBUG_TOGGLE_SHARD_ASSIGNMENT);
 if (v == null) {
@@ -92,6 +96,8 @@ public class BackdoorToggles {
 _backdoorToggles.remove();
 }
 
+public final static String KEY_QUERY_ID = "QUERY_ID";
+
 /**
  * set DEBUG_TOGGLE_DISABLE_FUZZY_KEY=true to disable fuzzy key for 
debug/profile usage
  *

http://git-wip-us.apache.org/repos/asf/kylin/blob/db09f5f9/core-common/src/main/java/org/apache/kylin/common/util/SetThreadName.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/util/SetThreadName.java 
b/core-common/src/main/java/org/apache/kylin/common/util/SetThreadName.java
new file mode 100644
index 000..9e5197e
--- /dev/null
+++ b/core-common/src/main/java/org/apache/kylin/common/util/SetThreadName.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.common.util;
+
+import java.io.Closeable;
+
+/**
+ *
+ * An utility that makes it easy to change current thread name in a 
try-with-resource block.
+ *
+ *  Example:
+ *
+ * 
+ * // old thread name
+ * try (SetThreadName ignored = new SetThreadName(format, arg1, arg2)) {
+ * // new thread name
+ * }
+ * // old thread name
+ * 
+ */
+public class SetThreadName implements Closeable {
+private final String originThreadName;
+
+public SetThreadName(String format, Object... args) {
+originThreadName = Thread.currentThread().getName();
+Thread.currentThread().setName(String.format(format, args) + "-" + 
Thread.currentThread().getId());
+}
+
+@Override
+public void close() {
+Thread.currentThread().setName(originThreadName);
+}
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/db09f5f9/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
--
diff --git 

[22/50] [abbrv] kylin git commit: KYLIN-2012 rename RealizationStatusEnum.DESCBROKEN to RealizationStatusEnum.BROKEN

[lidong]

KYLIN-2012 rename RealizationStatusEnum.DESCBROKEN to 
RealizationStatusEnum.BROKEN


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5974fc08
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5974fc08
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5974fc08

Branch: refs/heads/master-hbase1.x
Commit: 5974fc0870be17a8801c55c7496093d42dbb7c4f
Parents: 36cf99e
Author: gaodayue 
Authored: Thu Oct 13 17:16:02 2016 +0800
Committer: gaodayue 
Committed: Thu Oct 13 17:16:02 2016 +0800

--
 .../main/java/org/apache/kylin/cube/CubeDescManager.java  |  4 ++--
 .../src/main/java/org/apache/kylin/cube/CubeInstance.java |  2 +-
 .../src/main/java/org/apache/kylin/cube/CubeManager.java  |  6 +++---
 .../java/org/apache/kylin/cube/CubeManagerCacheTest.java  |  4 ++--
 .../kylin/metadata/realization/RealizationStatusEnum.java |  2 +-
 .../org/apache/kylin/rest/controller/CubeController.java  |  2 +-
 .../java/org/apache/kylin/rest/service/JobService.java|  2 +-
 webapp/app/partials/cubes/cubes.html  | 10 +-
 8 files changed, 16 insertions(+), 16 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/5974fc08/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
index d6364fe..602707f 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
@@ -158,10 +158,10 @@ public class CubeDescManager {
 cubeDescMap.putLocal(ndesc.getName(), ndesc);
 Cuboid.reloadCache(name);
 
-// if related cube is in DESCBROKEN state before, change it back to 
DISABLED
+// if related cube is in BROKEN state before, change it back to 
DISABLED
 CubeManager cubeManager = CubeManager.getInstance(config);
 for (CubeInstance cube : cubeManager.getCubesByDesc(name)) {
-if (cube.getStatus() == RealizationStatusEnum.DESCBROKEN) {
+if (cube.getStatus() == RealizationStatusEnum.BROKEN) {
 cubeManager.reloadCubeLocal(cube.getName());
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/5974fc08/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
index 720690d..b8e9394 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
@@ -153,7 +153,7 @@ public class CubeInstance extends RootPersistentEntity 
implements IRealization,
 // in a temporary broken state, so that user can edit and fix it. Broken 
state is often due to
 // schema changes at source.
 public boolean allowBrokenDescriptor() {
-return (getStatus() == RealizationStatusEnum.DISABLED  || getStatus() 
== RealizationStatusEnum.DESCBROKEN) && segments.isEmpty();
+return (getStatus() == RealizationStatusEnum.DISABLED  || getStatus() 
== RealizationStatusEnum.BROKEN) && segments.isEmpty();
 }
 
 public String getResourcePath() {

http://git-wip-us.apache.org/repos/asf/kylin/blob/5974fc08/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index c558c6d..1ebeb60 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -994,12 +994,12 @@ public class CubeManager implements IRealizationProvider {
 checkNotNull(cubeDesc, "cube descriptor '%s' (for cube '%s') not 
found", cube.getDescName(), cubeName);
 
 if (!cubeDesc.getError().isEmpty()) {
-cube.setStatus(RealizationStatusEnum.DESCBROKEN);
+cube.setStatus(RealizationStatusEnum.BROKEN);
 logger.warn("cube descriptor {} (for cube '{}') is broken", 
cubeDesc.getResourcePath(), cubeName);
 
-} else if (cube.getStatus() == RealizationStatusEnum.DESCBROKEN) {
+} else if (cube.getStatus() == RealizationStatusEnum.BROKEN) {
 cube.setStatus(RealizationStatusEnum.DISABLED);
-logger.info("cube {} changed from DESCBROKEN to DISABLED", 
cubeName);
+  

[04/50] [abbrv] kylin git commit: KYLIN-2072 Cleanup old streaming code

[lidong]

http://git-wip-us.apache.org/repos/asf/kylin/blob/5aee0226/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/StreamingManager.java
--
diff --git 
a/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/StreamingManager.java
 
b/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/StreamingManager.java
deleted file mode 100644
index 271bf41..000
--- 
a/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/StreamingManager.java
+++ /dev/null
@@ -1,248 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.engine.streaming;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.ConcurrentHashMap;
-
-import org.apache.commons.lang3.StringUtils;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.persistence.JsonSerializer;
-import org.apache.kylin.common.persistence.ResourceStore;
-import org.apache.kylin.common.persistence.Serializer;
-import org.apache.kylin.metadata.MetadataConstants;
-import org.apache.kylin.metadata.cachesync.Broadcaster;
-import org.apache.kylin.metadata.cachesync.Broadcaster.Event;
-import org.apache.kylin.metadata.cachesync.CaseInsensitiveStringCache;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- */
-public class StreamingManager {
-
-private static final Logger logger = 
LoggerFactory.getLogger(StreamingManager.class);
-
-// static cached instances
-private static final ConcurrentHashMap 
CACHE = new ConcurrentHashMap();
-
-public static final Serializer STREAMING_SERIALIZER = new 
JsonSerializer(StreamingConfig.class);
-
-private KylinConfig config;
-
-// name ==> StreamingConfig
-private CaseInsensitiveStringCache streamingMap;
-
-public static void clearCache() {
-CACHE.clear();
-}
-
-private StreamingManager(KylinConfig config) throws IOException {
-this.config = config;
-this.streamingMap = new 
CaseInsensitiveStringCache(config, "streaming");
-
-// touch lower level metadata before registering my listener
-reloadAllStreaming();
-Broadcaster.getInstance(config).registerListener(new 
StreamingSyncListener(), "streaming");
-}
-
-private class StreamingSyncListener extends Broadcaster.Listener {
-@Override
-public void onClearAll(Broadcaster broadcaster) throws IOException {
-clearCache();
-}
-
-@Override
-public void onEntityChange(Broadcaster broadcaster, String entity, 
Event event, String cacheKey) throws IOException {
-if (event == Event.DROP)
-removeStreamingLocal(cacheKey);
-else
-reloadStreamingConfigLocal(cacheKey);
-}
-}
-
-private ResourceStore getStore() {
-return ResourceStore.getStore(this.config);
-}
-
-public static StreamingManager getInstance(KylinConfig config) {
-StreamingManager r = CACHE.get(config);
-if (r != null) {
-return r;
-}
-
-synchronized (StreamingManager.class) {
-r = CACHE.get(config);
-if (r != null) {
-return r;
-}
-try {
-r = new StreamingManager(config);
-CACHE.put(config, r);
-if (CACHE.size() > 1) {
-logger.warn("More than one singleton exist");
-}
-return r;
-} catch (IOException e) {
-throw new IllegalStateException("Failed to init 
StreamingManager from " + config, e);
-}
-}
-}
-
-private static String formatStreamingConfigPath(String name) {
-return ResourceStore.STREAMING_RESOURCE_ROOT + "/" + name + ".json";
-}
-
-private static String formatStreamingOutputPath(String streaming, int 
partition) {
-return ResourceStore.STREAMING_OUTPUT_RESOURCE_ROOT + "/" + streaming 
+ "_" + partition + ".json";
-}
-
-private static String 

[14/50] [abbrv] kylin git commit: fix CI

[lidong]

fix CI


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/fad9e818
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/fad9e818
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/fad9e818

Branch: refs/heads/master-hbase1.x
Commit: fad9e818fe965e15dd086bb2c33d63784b347e44
Parents: 0c7e268
Author: shaofengshi 
Authored: Tue Oct 11 17:10:16 2016 +0800
Committer: shaofengshi 
Committed: Tue Oct 11 17:10:32 2016 +0800

--
 .../test/java/org/apache/kylin/provision/BuildCubeWithStream.java  | 2 ++
 1 file changed, 2 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/fad9e818/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java 
b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
index 65dd151..d2e46a5 100644
--- a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
+++ b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
@@ -317,8 +317,10 @@ public class BuildCubeWithStream {
 logger.info("Build is done");
 buildCubeWithStream.cleanup();
 logger.info("Going to exit");
+System.exit(0);
 } catch (Throwable e) {
 logger.error("error", e);
+System.exit(1);
 } finally {
 if (buildCubeWithStream != null) {
 buildCubeWithStream.after();

[25/50] [abbrv] kylin git commit: minor, enhance KYLIN_HOME setup

[lidong]

minor, enhance KYLIN_HOME setup


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/021ccfd0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/021ccfd0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/021ccfd0

Branch: refs/heads/master-hbase1.x
Commit: 021ccfd0984bd41e2f2286ebec83efb551899575
Parents: bebbc78
Author: Yang Li 
Authored: Sat Oct 15 15:58:48 2016 +0800
Committer: Yang Li 
Committed: Sat Oct 15 15:59:38 2016 +0800

--
 build/bin/kylin.sh | 7 ---
 build/bin/metastore.sh | 9 +
 2 files changed, 9 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/021ccfd0/build/bin/kylin.sh
--
diff --git a/build/bin/kylin.sh b/build/bin/kylin.sh
index da53d3d..9286055e 100644
--- a/build/bin/kylin.sh
+++ b/build/bin/kylin.sh
@@ -17,9 +17,10 @@
 # limitations under the License.
 #
 
-# We should set KYLIN_HOME here for multiple tomcat instances that are on the 
same node.
-# In addition, we should set a KYLIN_HOME for the global use as normal.
-KYLIN_HOME=`dirname $0`/..
+dir=$(cd -P -- "$(dirname -- "$0")" && pwd -P)
+
+# set KYLIN_HOME with consideration for multiple instances that are on the 
same node
+KYLIN_HOME=${KYLIN_HOME:-"${dir}/../"}
 export KYLIN_HOME=`cd "$KYLIN_HOME"; pwd`
 dir="$KYLIN_HOME/bin"
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/021ccfd0/build/bin/metastore.sh
--
diff --git a/build/bin/metastore.sh b/build/bin/metastore.sh
index 5f34bff..baf3d7a 100755
--- a/build/bin/metastore.sh
+++ b/build/bin/metastore.sh
@@ -24,11 +24,12 @@
 # take a look at SandboxMetastoreCLI
 
 
-dir=$(dirname ${0})
+dir=$(cd -P -- "$(dirname -- "$0")" && pwd -P)
 
-# We should set KYLIN_HOME here for multiple tomcat instsances that are on the 
same node.
-# In addition, we should set a KYLIN_HOME for the global use as normal.
-export KYLIN_HOME=${dir}/../
+# set KYLIN_HOME with consideration for multiple instances that are on the 
same node
+KYLIN_HOME=${KYLIN_HOME:-"${dir}/../"}
+export KYLIN_HOME=`cd "$KYLIN_HOME"; pwd`
+dir="$KYLIN_HOME/bin"
 
 source ${dir}/check-env.sh
 

[36/50] [abbrv] kylin git commit: KYLIN-2099 fix sample table/cube desc

[lidong]

KYLIN-2099 fix sample table/cube desc


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5429006a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5429006a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5429006a

Branch: refs/heads/master-hbase1.x
Commit: 5429006a5976a52a08ef39976dfedfeb164629ea
Parents: 3924672
Author: Li Yang 
Authored: Mon Oct 17 16:42:44 2016 +0800
Committer: Li Yang 
Committed: Mon Oct 17 16:45:31 2016 +0800

--
 examples/sample_cube/create_sample_tables.sql   | 16 
 .../template/cube_desc/kylin_sales_cube_desc.json   |  2 +-
 .../table/DEFAULT.KYLIN_CATEGORY_GROUPINGS.json |  4 ++--
 3 files changed, 11 insertions(+), 11 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/5429006a/examples/sample_cube/create_sample_tables.sql
--
diff --git a/examples/sample_cube/create_sample_tables.sql 
b/examples/sample_cube/create_sample_tables.sql
index fa5e25b..9fa17d7 100644
--- a/examples/sample_cube/create_sample_tables.sql
+++ b/examples/sample_cube/create_sample_tables.sql
@@ -116,10 +116,10 @@ CAL_DT date COMMENT 'Date, PK'
 ,YTM_YN_ID tinyint
 ,YTQ_YN_ID tinyint
 ,YTW_YN_ID tinyint
-,CRE_DATE string
-,CRE_USER string
-,UPD_DATE string
-,UPD_USER string
+,KYLIN_CAL_DT_CRE_DATE string
+,KYLIN_CAL_DT_CRE_USER string
+,KYLIN_CAL_DT_UPD_DATE string
+,KYLIN_CAL_DT_UPD_USER string
 )
 COMMENT 'Date Dimension Table'
 ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
@@ -137,10 +137,10 @@ LEAF_CATEG_ID bigint COMMENT 'Category ID, PK'
 ,REGN_CATEG string
 ,USER_DEFINED_FIELD1 string COMMENT 'User Defined Field1'
 ,USER_DEFINED_FIELD3 string COMMENT 'User Defined Field3'
-,CRE_DATE string
-,UPD_DATE string COMMENT 'Last Updated Date'
-,CRE_USER string
-,UPD_USER string COMMENT 'Last Updated User'
+,KYLIN_GROUPINGS_CRE_DATE string
+,KYLIN_GROUPINGS_UPD_DATE string COMMENT 'Last Updated Date'
+,KYLIN_GROUPINGS_CRE_USER string
+,KYLIN_GROUPINGS_UPD_USER string COMMENT 'Last Updated User'
 ,META_CATEG_ID decimal
 ,META_CATEG_NAME string COMMENT 'Level1 Category'
 ,CATEG_LVL2_ID decimal

http://git-wip-us.apache.org/repos/asf/kylin/blob/5429006a/examples/sample_cube/template/cube_desc/kylin_sales_cube_desc.json
--
diff --git a/examples/sample_cube/template/cube_desc/kylin_sales_cube_desc.json 
b/examples/sample_cube/template/cube_desc/kylin_sales_cube_desc.json
index cc1cd76..3e9bff4 100644
--- a/examples/sample_cube/template/cube_desc/kylin_sales_cube_desc.json
+++ b/examples/sample_cube/template/cube_desc/kylin_sales_cube_desc.json
@@ -14,7 +14,7 @@
 "name" : "CATEGORY",
 "table" : "DEFAULT.KYLIN_CATEGORY_GROUPINGS",
 "column" : "{FK}",
-"derived" : [ "USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3", "UPD_DATE", 
"UPD_USER" ]
+"derived" : [ "USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3" ]
   }, {
 "name" : "CATEGORY_HIERARCHY",
 "table" : "DEFAULT.KYLIN_CATEGORY_GROUPINGS",

http://git-wip-us.apache.org/repos/asf/kylin/blob/5429006a/examples/sample_cube/template/table/DEFAULT.KYLIN_CATEGORY_GROUPINGS.json
--
diff --git 
a/examples/sample_cube/template/table/DEFAULT.KYLIN_CATEGORY_GROUPINGS.json 
b/examples/sample_cube/template/table/DEFAULT.KYLIN_CATEGORY_GROUPINGS.json
index 8e97309..accba35 100644
--- a/examples/sample_cube/template/table/DEFAULT.KYLIN_CATEGORY_GROUPINGS.json
+++ b/examples/sample_cube/template/table/DEFAULT.KYLIN_CATEGORY_GROUPINGS.json
@@ -40,7 +40,7 @@
 "datatype" : "string"
   }, {
 "id" : "10",
-"name" : "UPD_DATE",
+"name" : "KYLIN_GROUPINGS_UPD_DATE",
 "datatype" : "string"
   }, {
 "id" : "11",
@@ -48,7 +48,7 @@
 "datatype" : "string"
   }, {
 "id" : "12",
-"name" : "UPD_USER",
+"name" : "KYLIN_GROUPINGS_UPD_USER",
 "datatype" : "string"
   }, {
 "id" : "13",

[01/50] [abbrv] kylin git commit: KYLIN-2072 further cleanup old streaming code [Forced Update!]

[lidong]

Repository: kylin
Updated Branches:
  refs/heads/master-hbase1.x 3d38f042e -> b8391565e (forced update)


KYLIN-2072 further cleanup old streaming code


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ed643e6b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ed643e6b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ed643e6b

Branch: refs/heads/master-hbase1.x
Commit: ed643e6b20e31a4c3a45d72dc8e5ff1287584764
Parents: c67fa74
Author: shaofengshi 
Authored: Sun Oct 9 22:16:38 2016 +0800
Committer: shaofengshi 
Committed: Mon Oct 10 13:32:44 2016 +0800

--
 .../kylin/common/util/StreamingBatch.java   | 44 
 .../kylin/common/util/StreamingMessage.java |  3 --
 .../org/apache/kylin/engine/EngineFactory.java  | 14 +--
 .../kylin/engine/IStreamingCubingEngine.java| 26 
 4 files changed, 1 insertion(+), 86 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/ed643e6b/core-common/src/main/java/org/apache/kylin/common/util/StreamingBatch.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/util/StreamingBatch.java 
b/core-common/src/main/java/org/apache/kylin/common/util/StreamingBatch.java
deleted file mode 100644
index e000aa6..000
--- a/core-common/src/main/java/org/apache/kylin/common/util/StreamingBatch.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.common.util;
-
-import java.util.List;
-
-/**
- */
-public final class StreamingBatch {
-
-private final List messages;
-
-private final Pair timeRange;
-
-public StreamingBatch(List messages, Pair 
timeRange) {
-this.messages = messages;
-this.timeRange = timeRange;
-}
-
-public List getMessages() {
-return messages;
-}
-
-public Pair getTimeRange() {
-return timeRange;
-}
-
-}

http://git-wip-us.apache.org/repos/asf/kylin/blob/ed643e6b/core-common/src/main/java/org/apache/kylin/common/util/StreamingMessage.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/util/StreamingMessage.java 
b/core-common/src/main/java/org/apache/kylin/common/util/StreamingMessage.java
index 53ab195..981c8a8 100644
--- 
a/core-common/src/main/java/org/apache/kylin/common/util/StreamingMessage.java
+++ 
b/core-common/src/main/java/org/apache/kylin/common/util/StreamingMessage.java
@@ -18,7 +18,6 @@
 
 package org.apache.kylin.common.util;
 
-import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 
@@ -34,8 +33,6 @@ public class StreamingMessage {
 
 private Map params;
 
-public static final StreamingMessage EOF = new 
StreamingMessage(Collections. emptyList(), 0L, 0L, Collections. emptyMap());
-
 public StreamingMessage(List data, long offset, long timestamp, 
Map params) {
 this.data = data;
 this.offset = offset;

http://git-wip-us.apache.org/repos/asf/kylin/blob/ed643e6b/core-job/src/main/java/org/apache/kylin/engine/EngineFactory.java
--
diff --git a/core-job/src/main/java/org/apache/kylin/engine/EngineFactory.java 
b/core-job/src/main/java/org/apache/kylin/engine/EngineFactory.java
index 7044a3e..acaa7da 100644
--- a/core-job/src/main/java/org/apache/kylin/engine/EngineFactory.java
+++ b/core-job/src/main/java/org/apache/kylin/engine/EngineFactory.java
@@ -31,23 +31,15 @@ import org.apache.kylin.metadata.model.IJoinedFlatTableDesc;
 public class EngineFactory {
 
 private static ImplementationSwitch batchEngines;
-private static ImplementationSwitch 
streamingEngines;
 static {
 Map impls = 
KylinConfig.getInstanceFromEnv().getJobEngines();
-

[49/50] [abbrv] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[lidong]

http://git-wip-us.apache.org/repos/asf/kylin/blob/b8391565/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-if (admin.isTableEnabled(table)) {
-admin.disableTable(table);
+if 
(admin.isTableEnabled(TableName.valueOf(table))) {
+admin.disableTable(TableName.valueOf(table));
 }
-admin.deleteTable(table);
+admin.deleteTable(TableName.valueOf(table));
 logger.debug("Dropped htable: " + table);
 output.append("HBase table " + table + " is 
dropped. \n");
 } else {

http://git-wip-us.apache.org/repos/asf/kylin/blob/b8391565/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index a150607..56f867a 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,9 +21,11 @@ package org.apache.kylin.storage.hbase.util;
 import java.io.IOException;
 
 import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.AbstractApplication;
 import org.apache.kylin.common.util.OptionsHelper;
 import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -38,8 +40,8 @@ public class CleanHtableCLI extends AbstractApplication {
 protected static final Logger logger = 
LoggerFactory.getLogger(CleanHtableCLI.class);
 
 private void clean() throws IOException {
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();
 
 for (HTableDescriptor descriptor : hbaseAdmin.listTables()) {
 String name = 

[43/50] [abbrv] kylin git commit: Ignore ITKylinQueryTest as it will be invoked by sub-classes

[lidong]

Ignore ITKylinQueryTest as it will be invoked by sub-classes

Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c84892f5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c84892f5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c84892f5

Branch: refs/heads/master-hbase1.x
Commit: c84892f527a6a16904d3cb801748e35248b4d90a
Parents: 03118bd
Author: shaofengshi 
Authored: Tue Oct 18 15:14:49 2016 +0800
Committer: shaofengshi 
Committed: Tue Oct 18 15:17:01 2016 +0800

--
 kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java | 1 +
 1 file changed, 1 insertion(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/c84892f5/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java 
b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
index ffd85e3..2ec5324 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
@@ -45,6 +45,7 @@ import org.junit.rules.ExpectedException;
 
 import com.google.common.collect.Maps;
 
+@Ignore("KylinQueryTest is contained by ITCombinationTest")
 public class ITKylinQueryTest extends KylinTestBase {
 
 @Rule

[39/50] [abbrv] kylin git commit: hive job use overrided MR job configuration by cube properties

[lidong]

hive job use overrided MR job configuration by cube properties

Signed-off-by: shaofengshi 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ac356f01
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ac356f01
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ac356f01

Branch: refs/heads/master-hbase1.x
Commit: ac356f014d52c4b13ad72e9d6a537e50e9ace5fb
Parents: c92f79a
Author: lijieliang 
Authored: Fri Oct 14 13:01:32 2016 +0800
Committer: shaofengshi 
Committed: Mon Oct 17 18:25:26 2016 +0800

--
 .../apache/kylin/source/hive/HiveMRInput.java   | 24 +---
 1 file changed, 21 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/ac356f01/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
--
diff --git 
a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java 
b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
index 202e480..2ec1fbb 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
@@ -35,6 +35,7 @@ import org.apache.kylin.common.util.CliCommandExecutor;
 import org.apache.kylin.common.util.Pair;
 import org.apache.kylin.cube.CubeInstance;
 import org.apache.kylin.cube.CubeManager;
+import org.apache.kylin.cube.CubeSegment;
 import org.apache.kylin.engine.mr.HadoopUtil;
 import org.apache.kylin.engine.mr.IMRInput;
 import org.apache.kylin.engine.mr.JobBuilderSupport;
@@ -60,6 +61,8 @@ import com.google.common.collect.Sets;
 
 public class HiveMRInput implements IMRInput {
 
+private static final String MR_OVERRIDE_JOB_QUEUENAME = 
"mapreduce.job.queuename";
+
 @Override
 public IMRBatchCubingInputSide 
getBatchCubingInputSide(IJoinedFlatTableDesc flatDesc) {
 return new BatchCubingInputSide(flatDesc);
@@ -154,7 +157,10 @@ public class HiveMRInput implements IMRInput {
 StringBuilder hiveInitBuf = new StringBuilder();
 hiveInitBuf.append("USE 
").append(conf.getConfig().getHiveDatabaseForIntermediateTable()).append(";\n");
 
hiveInitBuf.append(JoinedFlatTable.generateHiveSetStatements(conf));
-
+final KylinConfig kylinConfig = ((CubeSegment) 
flatTableDesc.getSegment()).getConfig();
+if 
(kylinConfig.getMRConfigOverride().get(MR_OVERRIDE_JOB_QUEUENAME) != null) {
+hiveInitBuf.append("SET 
mapreduce.job.queuename=").append(kylinConfig.getMRConfigOverride().get(MR_OVERRIDE_JOB_QUEUENAME)).append(";\n");
+}
 String rowCountOutputDir = 
JobBuilderSupport.getJobWorkingDir(conf, jobId) + "/row_count";
 
 RedistributeFlatHiveTableStep step = new 
RedistributeFlatHiveTableStep();
@@ -172,6 +178,11 @@ public class HiveMRInput implements IMRInput {
 final ShellExecutable step = new ShellExecutable();
 
 final HiveCmdBuilder hiveCmdBuilder = new HiveCmdBuilder();
+final KylinConfig kylinConfig = ((CubeSegment) 
flatTableDesc.getSegment()).getConfig();
+if 
(kylinConfig.getMRConfigOverride().get(MR_OVERRIDE_JOB_QUEUENAME) != null) {
+hiveCmdBuilder.addStatement("SET mapreduce.job.queuename =" +
+
kylinConfig.getMRConfigOverride().get(MR_OVERRIDE_JOB_QUEUENAME) + ";\n");
+}
 
hiveCmdBuilder.addStatement(JoinedFlatTable.generateHiveSetStatements(conf));
 hiveCmdBuilder.addStatement("set 
hive.exec.compress.output=false;\n");
 
hiveCmdBuilder.addStatement(JoinedFlatTable.generateCountDataStatement(flatTableDesc,
 rowCountOutputDir));
@@ -187,7 +198,7 @@ public class HiveMRInput implements IMRInput {
 
step.setName(ExecutableConstants.STEP_NAME_MATERIALIZE_HIVE_VIEW_IN_LOOKUP);
 HiveCmdBuilder hiveCmdBuilder = new HiveCmdBuilder();
 
-KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
+KylinConfig kylinConfig = ((CubeSegment) 
flatDesc.getSegment()).getConfig();
 MetadataManager metadataManager = 
MetadataManager.getInstance(kylinConfig);
 final Set lookupViewsTables = Sets.newHashSet();
 
@@ -201,6 +212,10 @@ public class HiveMRInput implements IMRInput {
 if (lookupViewsTables.size() == 0) {
 return null;
 }
+if 
(kylinConfig.getMRConfigOverride().get(MR_OVERRIDE_JOB_QUEUENAME) != null) {
+hiveCmdBuilder.addStatement("SET mapreduce.job.queuename =" +
+

[42/50] [abbrv] kylin git commit: KYLIN-2099 fix sample model

[lidong]

KYLIN-2099 fix sample model


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/03118bd4
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/03118bd4
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/03118bd4

Branch: refs/heads/master-hbase1.x
Commit: 03118bd445e7cfba14b75751fb9f478fe650b88a
Parents: c13ab1c
Author: Li Yang 
Authored: Tue Oct 18 12:36:43 2016 +0800
Committer: Li Yang 
Committed: Tue Oct 18 12:37:13 2016 +0800

--
 examples/sample_cube/template/model_desc/kylin_sales_model.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/03118bd4/examples/sample_cube/template/model_desc/kylin_sales_model.json
--
diff --git a/examples/sample_cube/template/model_desc/kylin_sales_model.json 
b/examples/sample_cube/template/model_desc/kylin_sales_model.json
index 13fc356..3c4fa5a 100644
--- a/examples/sample_cube/template/model_desc/kylin_sales_model.json
+++ b/examples/sample_cube/template/model_desc/kylin_sales_model.json
@@ -21,7 +21,7 @@
 "columns" : ["CAL_DT", "WEEK_BEG_DT"]
   }, {
 "table" : "DEFAULT.KYLIN_CATEGORY_GROUPINGS",
-"columns" : [ "USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3", "UPD_DATE", 
"UPD_USER", "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME" ]
+"columns" : [ "USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3", 
"META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME" ]
   }, {
 "table" : "DEFAULT.KYLIN_SALES",
 "columns" : [ "LSTG_FORMAT_NAME", "SELLER_ID", "PART_DT", "USER_ID", 
"REGION" ]

[28/50] [abbrv] kylin git commit: KYLIN-2088 Refactor MeasureType to allow mutliple UDAF defined on a measure type

[lidong]

KYLIN-2088 Refactor MeasureType to allow mutliple UDAF defined on a measure type


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0018a212
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0018a212
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0018a212

Branch: refs/heads/master-hbase1.x
Commit: 0018a212470bfe1e8edd3603d383da80f2ffd322
Parents: b4c970a
Author: Yang Li 
Authored: Sun Oct 16 23:44:49 2016 +0800
Committer: Yang Li 
Committed: Sun Oct 16 23:44:49 2016 +0800

--
 .../apache/kylin/common/KylinConfigBase.java|  1 -
 .../org/apache/kylin/measure/MeasureType.java   | 16 +++--
 .../kylin/measure/MeasureTypeFactory.java   | 46 +++---
 .../kylin/measure/basic/BasicMeasureType.java   |  6 --
 .../kylin/measure/bitmap/BitmapMeasureType.java | 43 -
 .../dim/DimCountDistinctMeasureType.java| 10 ++-
 .../ExtendedColumnMeasureType.java  |  6 +-
 .../kylin/measure/hllc/HLLCMeasureType.java | 10 ++-
 .../kylin/measure/raw/RawMeasureType.java   |  6 +-
 .../kylin/measure/topn/TopNMeasureType.java |  5 --
 .../kylin/metadata/realization/SQLDigest.java   | 43 -
 .../kylin/engine/mr/steps/CubeReducerTest.java  |  5 +-
 .../kylin/storage/hbase/ITStorageTest.java  |  3 +-
 .../kylin/query/relnode/ColumnRowType.java  |  7 +++
 .../kylin/query/relnode/OLAPAggregateRel.java   | 66 +++-
 .../apache/kylin/query/relnode/OLAPContext.java | 15 +++--
 .../kylin/query/schema/OLAPSchemaFactory.java   | 13 +++-
 .../cube/MeasureTypeOnlyAggrInBaseTest.java |  5 --
 18 files changed, 187 insertions(+), 119 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/0018a212/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 7dacd06..4942081 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -766,7 +766,6 @@ abstract public class KylinConfigBase implements 
Serializable {
 
 public Map getUDFs() {
 Map udfMap = getPropertiesByPrefix("kylin.query.udf.");
-udfMap.put("intersect_count", 
"org.apache.kylin.measure.bitmap.BitmapIntersectDistinctCountAggFunc");
 return udfMap;
 }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/0018a212/core-metadata/src/main/java/org/apache/kylin/measure/MeasureType.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureType.java 
b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureType.java
index e7312f2..de1b442 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureType.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureType.java
@@ -102,22 +102,20 @@ abstract public class MeasureType {
  * Query Rewrite
  * 
 */
 
-// TODO support user defined Calcite aggr function
-
 /** Whether or not Calcite rel-tree needs rewrite to do last around of 
aggregation */
 abstract public boolean needRewrite();
 
-/** Does the rewrite involves an extra field for the pre-calculated */
+/** Does the rewrite involves an extra field to hold the pre-calculated */
 public boolean needRewriteField() {
 return true;
 }
 
-/** Returns a Calcite aggregation function implementation class */
-abstract public Class getRewriteCalciteAggrFunctionClass();
-
-/** Some measure may return different class depends on call name, eg. 
BitmapMeasureType */
-public Class getRewriteCalciteAggrFunctionClass(String callName) {
-return getRewriteCalciteAggrFunctionClass();
+/**
+ * Returns a map from UDAF to Calcite aggregation function implementation 
class.
+ * There can be zero or more UDAF defined on a measure type.
+ */
+public Map getRewriteCalciteAggrFunctions() {
+return null;
 }
 
 /* 


http://git-wip-us.apache.org/repos/asf/kylin/blob/0018a212/core-metadata/src/main/java/org/apache/kylin/measure/MeasureTypeFactory.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureTypeFactory.java 

[09/50] [abbrv] kylin git commit: Revert "KYLIN-2078 Can't see generated SQL at Web UI"

[lidong]

Revert "KYLIN-2078 Can't see generated SQL at Web UI"

This reverts commit 94a9f381548ff6ecfb253e189a3319252f18d16c.


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/dc969cd5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/dc969cd5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/dc969cd5

Branch: refs/heads/master-hbase1.x
Commit: dc969cd5cf7c37b1e0df6b8b6c620b9b9817754d
Parents: 94a9f38
Author: shaofengshi 
Authored: Mon Oct 10 17:54:08 2016 +0800
Committer: shaofengshi 
Committed: Mon Oct 10 17:54:08 2016 +0800

--
 .../java/org/apache/kylin/rest/controller/CubeController.java| 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/dc969cd5/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
 
b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index bb7dc6a..be242c3 100644
--- 
a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++ 
b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -40,6 +40,7 @@ import org.apache.kylin.engine.EngineFactory;
 import org.apache.kylin.job.JobInstance;
 import org.apache.kylin.job.JoinedFlatTable;
 import org.apache.kylin.metadata.model.IJoinedFlatTableDesc;
+import org.apache.kylin.metadata.model.SegmentStatusEnum;
 import org.apache.kylin.metadata.project.ProjectInstance;
 import org.apache.kylin.metadata.realization.RealizationStatusEnum;
 import org.apache.kylin.rest.exception.BadRequestException;
@@ -141,7 +142,8 @@ public class CubeController extends BasicController {
 @ResponseBody
 public GeneralResponse getSql(@PathVariable String cubeName, @PathVariable 
String segmentName) {
 CubeInstance cube = cubeService.getCubeManager().getCube(cubeName);
-IJoinedFlatTableDesc flatTableDesc = 
EngineFactory.getJoinedFlatTableDesc(cube.getDescriptor());
+CubeSegment cubeSegment = cube.getSegment(segmentName, 
SegmentStatusEnum.READY);
+IJoinedFlatTableDesc flatTableDesc = 
EngineFactory.getJoinedFlatTableDesc(cubeSegment);
 String sql = 
JoinedFlatTable.generateSelectDataStatement(flatTableDesc, false);
 
 GeneralResponse repsonse = new GeneralResponse();

[17/50] [abbrv] kylin git commit: KYLIN-2083 more RAM estimation test for MeasureAggregator and GTAggregateScanner

[lidong]

KYLIN-2083 more RAM estimation test for MeasureAggregator and GTAggregateScanner


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/460536c0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/460536c0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/460536c0

Branch: refs/heads/master-hbase1.x
Commit: 460536c001575bd09e9f059062efe0d1049d6b31
Parents: ff85103
Author: gaodayue 
Authored: Wed Sep 28 17:09:06 2016 +0800
Committer: gaodayue 
Committed: Tue Oct 11 20:36:11 2016 +0800

--
 core-cube/pom.xml   |   5 +
 .../gridtable/AggregationCacheMemSizeTest.java  | 290 +--
 core-metadata/pom.xml   |   5 +
 .../measure/AggregatorMemEstimateTest.java  | 104 +++
 pom.xml |  35 ++-
 5 files changed, 292 insertions(+), 147 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/460536c0/core-cube/pom.xml
--
diff --git a/core-cube/pom.xml b/core-cube/pom.xml
index 7c9a549..39bba59 100644
--- a/core-cube/pom.xml
+++ b/core-cube/pom.xml
@@ -69,6 +69,11 @@
 junit
 test
 
+
+com.github.jbellis
+jamm
+test
+
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/460536c0/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java
--
diff --git 
a/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java
 
b/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java
index fcd434e..00c0bd0 100644
--- 
a/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java
+++ 
b/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java
@@ -17,197 +17,195 @@
 
 package org.apache.kylin.gridtable;
 
-import java.math.BigDecimal;
-import java.util.Comparator;
-import java.util.Random;
-import java.util.SortedMap;
-import java.util.TreeMap;
-
+import com.google.common.base.Stopwatch;
 import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.measure.MeasureAggregator;
 import org.apache.kylin.measure.basic.BigDecimalSumAggregator;
 import org.apache.kylin.measure.basic.DoubleSumAggregator;
 import org.apache.kylin.measure.basic.LongSumAggregator;
+import org.apache.kylin.measure.bitmap.BitmapAggregator;
+import org.apache.kylin.measure.bitmap.BitmapCounter;
 import org.apache.kylin.measure.hllc.HLLCAggregator;
 import org.apache.kylin.measure.hllc.HyperLogLogPlusCounter;
 import org.apache.kylin.metadata.datatype.DoubleMutable;
 import org.apache.kylin.metadata.datatype.LongMutable;
+import org.github.jamm.MemoryMeter;
 import org.junit.Test;
 
+import java.math.BigDecimal;
+import java.util.*;
+
 public class AggregationCacheMemSizeTest {
+private static final MemoryMeter meter = new MemoryMeter();
+private static final BitmapCounter[] bitmaps = new BitmapCounter[5];
+private static final Random random = new Random();
+
+// consider bitmaps with variant cardinality
+static {
+for (int i = 0; i < bitmaps.length; i++) {
+bitmaps[i] = new BitmapCounter();
+}
 
-public static final int NUM_OF_OBJS = 100 / 2;
+final int totalBits = 1_000_000;
 
-interface CreateAnObject {
-Object create();
-}
+// case 0: sparse, low-cardinality bitmap
+for (int i = 0; i < 100; i++) {
+bitmaps[0].add(random.nextInt(totalBits));
+}
 
-@Test
-public void testHLLCAggregatorSize() throws InterruptedException {
-int est = estimateObjectSize(new CreateAnObject() {
-@Override
-public Object create() {
-HLLCAggregator aggr = new HLLCAggregator(10);
-aggr.aggregate(new HyperLogLogPlusCounter(10));
-return aggr;
+// case 1: 20% full bitmap
+for (int i = 0; i < totalBits; i++) {
+if (random.nextInt(100) < 20) {
+bitmaps[1].add(i);
 }
-});
-System.out.println("HLLC: " + est);
-}
+}
 
-@Test
-public void testBigDecimalAggregatorSize() throws InterruptedException {
-int est = estimateObjectSize(new CreateAnObject() {
-@Override
-public Object create() {
-return newBigDecimalAggr();
+// case 2: half full bitmap
+for (int i = 0; i < totalBits; i++) {
+if (random.nextInt(100) < 50) {
+bitmaps[2].add(i);
 }
+}
 
-});
-

[34/50] [abbrv] kylin git commit: minor, better name for hive build step

[lidong]

minor, better name for hive build step


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ef0c9e5c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ef0c9e5c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ef0c9e5c

Branch: refs/heads/master-hbase1.x
Commit: ef0c9e5c64d5fbb1aebc415755b8f2b9ff8d1a83
Parents: 44cf9fb
Author: Li Yang 
Authored: Mon Oct 17 13:38:42 2016 +0800
Committer: Li Yang 
Committed: Mon Oct 17 13:39:06 2016 +0800

--
 .../java/org/apache/kylin/job/constant/ExecutableConstants.java| 1 +
 .../src/main/java/org/apache/kylin/source/hive/HiveMRInput.java| 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/ef0c9e5c/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java
--
diff --git 
a/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java 
b/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java
index 893c034..ad0b1b1 100644
--- 
a/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java
+++ 
b/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java
@@ -51,6 +51,7 @@ public final class ExecutableConstants {
 public static final String STEP_NAME_SAVE_STATISTICS = "Save Cuboid 
Statistics";
 public static final String STEP_NAME_MERGE_CUBOID = "Merge Cuboid Data";
 public static final String STEP_NAME_UPDATE_CUBE_INFO = "Update Cube Info";
+public static final String STEP_NAME_HIVE_CLEANUP = "Hive Cleanup";
 public static final String STEP_NAME_GARBAGE_COLLECTION = "Garbage 
Collection";
 public static final String STEP_NAME_GARBAGE_COLLECTION_HDFS = "Garbage 
Collection on HDFS";
 public static final String STEP_NAME_BUILD_II = "Build Inverted Index";

http://git-wip-us.apache.org/repos/asf/kylin/blob/ef0c9e5c/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
--
diff --git 
a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java 
b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
index 09ac522..202e480 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
@@ -244,7 +244,7 @@ public class HiveMRInput implements IMRInput {
 @Override
 public void addStepPhase4_Cleanup(DefaultChainedExecutable jobFlow) {
 GarbageCollectionStep step = new GarbageCollectionStep();
-step.setName(ExecutableConstants.STEP_NAME_GARBAGE_COLLECTION);
+step.setName(ExecutableConstants.STEP_NAME_HIVE_CLEANUP);
 step.setIntermediateTableIdentity(getIntermediateTableIdentity());
 step.setExternalDataPath(JoinedFlatTable.getTableDir(flatDesc, 
JobBuilderSupport.getJobWorkingDir(conf, jobFlow.getId(;
 
step.setHiveViewIntermediateTableIdentities(hiveViewIntermediateTables);

[47/50] [abbrv] kylin git commit: KYLIN-1839 minor refactor

[lidong]

KYLIN-1839 minor refactor


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6cafc422
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6cafc422
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6cafc422

Branch: refs/heads/master-hbase1.x
Commit: 6cafc422bc0ca2fa19a4516bcb4388719c99184c
Parents: aa1550c
Author: Yang Li 
Authored: Wed Oct 19 08:19:35 2016 +0800
Committer: Yang Li 
Committed: Wed Oct 19 08:30:49 2016 +0800

--
 .../kylin/engine/mr/common/AbstractHadoopJob.java  | 13 +++--
 1 file changed, 3 insertions(+), 10 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/6cafc422/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
index f70e3bb..a5b2d2e 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
@@ -318,7 +318,7 @@ public abstract class AbstractHadoopJob extends Configured 
implements Tool {
 }
 
 if (fs.getFileStatus(p).isDirectory()) {
-appendTmpDir(job, fs, p);
+appendTmpDir(job, fs, p, jarList, fileList);
 continue;
 }
 
@@ -335,18 +335,14 @@ public abstract class AbstractHadoopJob extends 
Configured implements Tool {
 }
 }
 
-private void appendTmpDir(Job job, FileSystem fs, Path tmpDir) {
+private void appendTmpDir(Job job, FileSystem fs, Path tmpDir, 
StringBuilder jarList, StringBuilder fileList) {
 try {
-Configuration jobConf = job.getConfiguration();
 FileStatus[] fList = fs.listStatus(tmpDir);
 
-StringBuilder jarList = new StringBuilder();
-StringBuilder fileList = new StringBuilder();
-
 for (FileStatus file : fList) {
 Path p = file.getPath();
 if (fs.getFileStatus(p).isDirectory()) {
-appendTmpDir(job, fs, p);
+appendTmpDir(job, fs, p, jarList, fileList);
 continue;
 }
 
@@ -356,9 +352,6 @@ public abstract class AbstractHadoopJob extends Configured 
implements Tool {
 list.append(fs.getFileStatus(p).getPath().toString());
 }
 
-appendTmpFiles(fileList.toString(), jobConf);
-appendTmpJars(jarList.toString(), jobConf);
-
 } catch (IOException e) {
 throw new RuntimeException(e);
 }

[26/50] [abbrv] kylin git commit: minor, move timezone setting early to surefire command line

[lidong]

minor, move timezone setting early to surefire command line


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/61a08d42
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/61a08d42
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/61a08d42

Branch: refs/heads/master-hbase1.x
Commit: 61a08d429a2c7419751b312e41bcee82048716e2
Parents: 021ccfd
Author: Yang Li 
Authored: Sun Oct 16 07:59:33 2016 +0800
Committer: Yang Li 
Committed: Sun Oct 16 07:59:33 2016 +0800

--
 .../src/test/java/org/apache/kylin/query/ITKylinQueryTest.java| 3 ---
 pom.xml   | 2 +-
 2 files changed, 1 insertion(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/61a08d42/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java 
b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
index 93d47f1..59a3a04 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
@@ -24,7 +24,6 @@ import java.io.File;
 import java.sql.SQLException;
 import java.util.List;
 import java.util.Map;
-import java.util.TimeZone;
 
 import org.apache.commons.lang3.StringUtils;
 import org.apache.kylin.common.KylinConfig;
@@ -66,8 +65,6 @@ public class ITKylinQueryTest extends KylinTestBase {
 
 
RemoveBlackoutRealizationsRule.blackList.add("CUBE[name=test_kylin_cube_with_view_left_join_empty]");
 
RemoveBlackoutRealizationsRule.blackList.add("CUBE[name=test_kylin_cube_with_view_inner_join_empty]");
-
-TimeZone.setDefault(TimeZone.getTimeZone("UTC"));
 }
 
 @AfterClass

http://git-wip-us.apache.org/repos/asf/kylin/blob/61a08d42/pom.xml
--
diff --git a/pom.xml b/pom.xml
index caa09ec..c7538e4 100644
--- a/pom.xml
+++ b/pom.xml
@@ -124,7 +124,7 @@
 
 
 
--Xms1G -Xmx2G -XX:MaxPermSize=512M
+-Xms1G -Xmx2G -XX:MaxPermSize=512M 
-Duser.timezone=UTC
 
 
 

[24/50] [abbrv] kylin git commit: Revert "KYLIN-2012 rename RealizationStatusEnum.DESCBROKEN to RealizationStatusEnum.BROKEN"

[lidong]

Revert "KYLIN-2012 rename RealizationStatusEnum.DESCBROKEN to 
RealizationStatusEnum.BROKEN"

This reverts commit 5974fc0870be17a8801c55c7496093d42dbb7c4f.


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/bebbc784
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/bebbc784
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/bebbc784

Branch: refs/heads/master-hbase1.x
Commit: bebbc78476c6b41b50fc0492134a5418d522533e
Parents: e6698c8
Author: shaofengshi 
Authored: Fri Oct 14 11:46:01 2016 +0800
Committer: shaofengshi 
Committed: Fri Oct 14 11:46:01 2016 +0800

--
 .../main/java/org/apache/kylin/cube/CubeDescManager.java  |  4 ++--
 .../src/main/java/org/apache/kylin/cube/CubeInstance.java |  2 +-
 .../src/main/java/org/apache/kylin/cube/CubeManager.java  |  6 +++---
 .../java/org/apache/kylin/cube/CubeManagerCacheTest.java  |  4 ++--
 .../kylin/metadata/realization/RealizationStatusEnum.java |  2 +-
 .../org/apache/kylin/rest/controller/CubeController.java  |  2 +-
 .../java/org/apache/kylin/rest/service/JobService.java|  2 +-
 webapp/app/partials/cubes/cubes.html  | 10 +-
 8 files changed, 16 insertions(+), 16 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/bebbc784/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
index 602707f..d6364fe 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
@@ -158,10 +158,10 @@ public class CubeDescManager {
 cubeDescMap.putLocal(ndesc.getName(), ndesc);
 Cuboid.reloadCache(name);
 
-// if related cube is in BROKEN state before, change it back to 
DISABLED
+// if related cube is in DESCBROKEN state before, change it back to 
DISABLED
 CubeManager cubeManager = CubeManager.getInstance(config);
 for (CubeInstance cube : cubeManager.getCubesByDesc(name)) {
-if (cube.getStatus() == RealizationStatusEnum.BROKEN) {
+if (cube.getStatus() == RealizationStatusEnum.DESCBROKEN) {
 cubeManager.reloadCubeLocal(cube.getName());
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/bebbc784/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
index b8e9394..720690d 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
@@ -153,7 +153,7 @@ public class CubeInstance extends RootPersistentEntity 
implements IRealization,
 // in a temporary broken state, so that user can edit and fix it. Broken 
state is often due to
 // schema changes at source.
 public boolean allowBrokenDescriptor() {
-return (getStatus() == RealizationStatusEnum.DISABLED  || getStatus() 
== RealizationStatusEnum.BROKEN) && segments.isEmpty();
+return (getStatus() == RealizationStatusEnum.DISABLED  || getStatus() 
== RealizationStatusEnum.DESCBROKEN) && segments.isEmpty();
 }
 
 public String getResourcePath() {

http://git-wip-us.apache.org/repos/asf/kylin/blob/bebbc784/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index 1ebeb60..c558c6d 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -994,12 +994,12 @@ public class CubeManager implements IRealizationProvider {
 checkNotNull(cubeDesc, "cube descriptor '%s' (for cube '%s') not 
found", cube.getDescName(), cubeName);
 
 if (!cubeDesc.getError().isEmpty()) {
-cube.setStatus(RealizationStatusEnum.BROKEN);
+cube.setStatus(RealizationStatusEnum.DESCBROKEN);
 logger.warn("cube descriptor {} (for cube '{}') is broken", 
cubeDesc.getResourcePath(), cubeName);
 
-} else if (cube.getStatus() == RealizationStatusEnum.BROKEN) {
+} else if (cube.getStatus() == RealizationStatusEnum.DESCBROKEN) {
 cube.setStatus(RealizationStatusEnum.DISABLED);
-

[29/50] [abbrv] kylin git commit: KYLIN-2088 fix CI

[lidong]

KYLIN-2088 fix CI


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/81403c1b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/81403c1b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/81403c1b

Branch: refs/heads/master-hbase1.x
Commit: 81403c1b879f23760d91e2fc0864ece17da00d00
Parents: 0018a21
Author: Yang Li 
Authored: Mon Oct 17 05:54:45 2016 +0800
Committer: Yang Li 
Committed: Mon Oct 17 05:54:45 2016 +0800

--
 .../java/org/apache/kylin/query/ITKylinQueryTest.java | 14 --
 1 file changed, 8 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/81403c1b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java 
b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
index a0706ca..ffd85e3 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
@@ -248,6 +248,14 @@ public class ITKylinQueryTest extends KylinTestBase {
 }
 
 @Test
+public void testIntersectCountQuery() throws Exception {
+// cannot compare coz H2 does not support intersect count yet..
+if ("left".equalsIgnoreCase(joinType)) {
+this.batchExecuteQuery(getQueryFolderPrefix() + 
"src/test/resources/query/sql_intersect_count");
+}
+}
+
+@Test
 public void testDimDistinctCountQuery() throws Exception {
 execAndCompQuery(getQueryFolderPrefix() + 
"src/test/resources/query/sql_distinct_dim", null, true);
 }
@@ -355,10 +363,4 @@ public class ITKylinQueryTest extends KylinTestBase {
 this.batchExecuteQuery(getQueryFolderPrefix() + 
"src/test/resources/query/sql_window");
 }
 
-@Test
-public void testIntersectCountQuery() throws Exception {
-// cannot compare coz H2 does not support intersect count yet..
-this.batchExecuteQuery(getQueryFolderPrefix() + 
"src/test/resources/query/sql_intersect_count");
-}
-
 }

[32/50] [abbrv] kylin git commit: KYLIN-2084 Unload sample table failed

[lidong]

KYLIN-2084 Unload sample table failed

Signed-off-by: Jason 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ef44b7f6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ef44b7f6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ef44b7f6

Branch: refs/heads/master-hbase1.x
Commit: ef44b7f6873b2e5ac55243818877474cdc0376d8
Parents: c3bfd77
Author: Jason 
Authored: Mon Oct 17 13:18:57 2016 +0800
Committer: Jason 
Committed: Mon Oct 17 13:20:16 2016 +0800

--
 .../java/org/apache/kylin/metadata/MetadataManager.java   | 10 ++
 .../org/apache/kylin/rest/controller/TableController.java |  3 +++
 .../java/org/apache/kylin/rest/service/ModelService.java  |  7 +++
 3 files changed, 20 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/ef44b7f6/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java 
b/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
index 92d41b9..e0c78ce 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
@@ -496,6 +496,16 @@ public class MetadataManager {
 return false;
 }
 
+public List getModelsUsingTable(String tableName, String 
projectName) throws IOException {
+List models = new ArrayList<>();
+for (DataModelDesc modelDesc : getModels(projectName)) {
+if (modelDesc.getAllTables().contains(tableName.toUpperCase())) {
+models.add(modelDesc.getName());
+}
+}
+return models;
+}
+
 public boolean isTableInAnyModel(String tableName) {
 for (DataModelDesc modelDesc : getModels()) {
 if (modelDesc.getAllTables().contains(tableName.toUpperCase())) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/ef44b7f6/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
 
b/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
index 34cc57f..d9050c1 100644
--- 
a/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
+++ 
b/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
@@ -197,6 +197,9 @@ public class TableController extends BasicController {
 if (!modelService.isTableInModel(tableName, project)) {
 cubeMgmtService.removeTableFromProject(tableName, project);
 rtn = true;
+}else{
+List models = 
modelService.getModelsUsingTable(tableName, project);
+throw new InternalErrorException("Table is already in use by 
models "+models);
 }
 } catch (IOException e) {
 logger.error(e.getMessage(), e);

http://git-wip-us.apache.org/repos/asf/kylin/blob/ef44b7f6/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java 
b/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java
index bfff619..2f8667c 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java
@@ -132,4 +132,11 @@ public class ModelService extends BasicService {
 tableName = dbTableName[0] + "." + dbTableName[1];
 return getMetadataManager().isTableInModel(tableName, projectName);
 }
+
+@PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#desc, 
'ADMINISTRATION') or hasPermission(#desc, 'MANAGEMENT')")
+public List getModelsUsingTable(String tableName, String 
projectName) throws IOException {
+String[] dbTableName = HadoopUtil.parseHiveTableName(tableName);
+tableName = dbTableName[0] + "." + dbTableName[1];
+return getMetadataManager().getModelsUsingTable(tableName, 
projectName);
+}
 }

[03/50] [abbrv] kylin git commit: KYLIN-2072 Cleanup old streaming code

[lidong]

http://git-wip-us.apache.org/repos/asf/kylin/blob/5aee0226/source-kafka/src/main/java/org/apache/kylin/source/kafka/diagnose/TimeHistogram.java
--
diff --git 
a/source-kafka/src/main/java/org/apache/kylin/source/kafka/diagnose/TimeHistogram.java
 
b/source-kafka/src/main/java/org/apache/kylin/source/kafka/diagnose/TimeHistogram.java
deleted file mode 100644
index 1c579c6..000
--- 
a/source-kafka/src/main/java/org/apache/kylin/source/kafka/diagnose/TimeHistogram.java
+++ /dev/null
@@ -1,85 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.source.kafka.diagnose;
-
-import java.util.concurrent.atomic.AtomicLong;
-
-public class TimeHistogram {
-private long[] bucketsBoundary;
-private AtomicLong[] counters;
-private String id;
-
-private static Object printLock = new Object();
-
-/**
- * example: [10,20] will generate three  buckets: (-∞,10), 
[10,20),[20,+∞)
- * unit: second
- */
-public TimeHistogram(long[] bucketsBoundary, String id) {
-this.bucketsBoundary = bucketsBoundary;
-this.counters = new AtomicLong[this.bucketsBoundary.length + 1];
-for (int i = 0; i < counters.length; i++) {
-this.counters[i] = new AtomicLong();
-}
-this.id = id;
-}
-
-/**
- * @param second in seconds
- */
-public void process(long second) {
-for (int i = 0; i < bucketsBoundary.length; ++i) {
-if (second < bucketsBoundary[i]) {
-counters[i].incrementAndGet();
-return;
-}
-}
-
-counters[bucketsBoundary.length].incrementAndGet();
-}
-
-/**
- * @param millis in milli seconds
- */
-public void processMillis(long millis) {
-process(millis / 1000);
-}
-
-public void printStatus() {
-long[] countersSnapshot = new long[counters.length];
-for (int i = 0; i < countersSnapshot.length; i++) {
-countersSnapshot[i] = counters[i].get();
-}
-
-long sum = 0;
-for (long counter : countersSnapshot) {
-sum += counter;
-}
-
-synchronized (printLock) {
-System.out.println("== status of TimeHistogram " + id 
+ " =");
-
-for (int i = 0; i < countersSnapshot.length; ++i) {
-System.out.println(String.format("bucket: %d , count: %d 
,percentage: %.4f", i, countersSnapshot[i], 1.0 * countersSnapshot[i] / (sum == 
0 ? 1 : sum)));
-}
-
-}
-}
-
-}

http://git-wip-us.apache.org/repos/asf/kylin/blob/5aee0226/source-kafka/src/main/java/org/apache/kylin/source/kafka/util/ByteBufferBackedInputStream.java
--
diff --git 
a/source-kafka/src/main/java/org/apache/kylin/source/kafka/util/ByteBufferBackedInputStream.java
 
b/source-kafka/src/main/java/org/apache/kylin/source/kafka/util/ByteBufferBackedInputStream.java
new file mode 100644
index 000..7a42598
--- /dev/null
+++ 
b/source-kafka/src/main/java/org/apache/kylin/source/kafka/util/ByteBufferBackedInputStream.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+package org.apache.kylin.source.kafka.util;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.ByteBuffer;
+
+/**
+ */

[10/50] [abbrv] kylin git commit: KYLIN-1726 segment status may have been updated while job status wasn't committed

[lidong]

KYLIN-1726 segment status may have been updated while job status wasn't 
committed


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/726b0ec9
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/726b0ec9
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/726b0ec9

Branch: refs/heads/master-hbase1.x
Commit: 726b0ec93f9f3b1b5da3520be385d6a2047d32c6
Parents: dc969cd
Author: shaofengshi 
Authored: Mon Oct 10 17:55:19 2016 +0800
Committer: shaofengshi 
Committed: Mon Oct 10 17:55:43 2016 +0800

--
 core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/726b0ec9/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index 269b324..c558c6d 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -836,8 +836,9 @@ public class CubeManager implements IRealizationProvider {
 if (StringUtils.isBlank(newSegment.getLastBuildJobID()))
 throw new IllegalStateException("For cube " + cube + ", segment " 
+ newSegment + " missing LastBuildJobID");
 
-if (isReady(newSegment) == true)
-throw new IllegalStateException("For cube " + cube + ", segment " 
+ newSegment + " state should be NEW but is READY");
+if (isReady(newSegment) == true) {
+logger.warn("For cube " + cube + ", segment " + newSegment + " 
state should be NEW but is READY");
+}
 
 List tobe = calculateToBeSegments(cube, newSegment);
 

[21/50] [abbrv] kylin git commit: KYLIN-2012 adjust rules for LookupTable & remove dead methods

[lidong]

KYLIN-2012 adjust rules for LookupTable & remove dead methods


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/36cf99ef
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/36cf99ef
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/36cf99ef

Branch: refs/heads/master-hbase1.x
Commit: 36cf99ef77486c1361a31f3e1f748bb277eca217
Parents: 96935fe
Author: gaodayue 
Authored: Thu Oct 13 16:56:05 2016 +0800
Committer: gaodayue 
Committed: Thu Oct 13 16:56:35 2016 +0800

--
 .../apache/kylin/metadata/MetadataManager.java  | 30 
 .../apache/kylin/source/hive/SchemaChecker.java | 27 ++
 2 files changed, 21 insertions(+), 36 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/36cf99ef/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java 
b/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
index 4e9c1ed..92d41b9 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
@@ -461,11 +461,6 @@ public class MetadataManager {
 reloadSourceTableAt(TableDesc.concatResourcePath(tableIdentity));
 }
 
-public void reloadTableCache(String tableIdentity) throws IOException {
-reloadSourceTableExt(tableIdentity);
-reloadSourceTable(tableIdentity);
-}
-
 public DataModelDesc getDataModelDesc(String name) {
 return dataModelDescMap.get(name);
 }
@@ -622,29 +617,4 @@ public class MetadataManager {
 srcTableExdMap.remove(tableIdentity);
 }
 
-public String appendDBName(String table) {
-
-if (table.indexOf(".") > 0)
-return table;
-
-Map map = getAllTablesMap();
-
-int count = 0;
-String result = null;
-for (TableDesc t : map.values()) {
-if (t.getName().equalsIgnoreCase(table)) {
-result = t.getIdentity();
-count++;
-}
-}
-
-if (count == 1)
-return result;
-
-if (count > 1) {
-logger.warn("There are more than 1 table named with '" + table + 
"' in different database; The program couldn't determine, randomly pick '" + 
result + "'");
-}
-return result;
-}
-
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/36cf99ef/source-hive/src/main/java/org/apache/kylin/source/hive/SchemaChecker.java
--
diff --git 
a/source-hive/src/main/java/org/apache/kylin/source/hive/SchemaChecker.java 
b/source-hive/src/main/java/org/apache/kylin/source/hive/SchemaChecker.java
index e955410..319ebee 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/SchemaChecker.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/SchemaChecker.java
@@ -152,7 +152,14 @@ public class SchemaChecker {
 }
 }
 
-private List checkAllUsedColumns(CubeInstance cube, TableDesc 
table, Map fieldsMap) {
+/**
+ * check whether all columns used in `cube` has compatible schema in 
current hive schema denoted by `fieldsMap`.
+ * @param cube cube to check, must use `table` in its model
+ * @param table kylin's table metadata
+ * @param fieldsMap current hive schema of `table`
+ * @return true if all columns used in `cube` has compatible schema with 
`fieldsMap`, false otherwise
+ */
+private List checkAllColumnsInCube(CubeInstance cube, TableDesc 
table, Map fieldsMap) {
 Set usedColumns = Sets.newHashSet();
 for (TblColRef col : cube.getAllColumns()) {
 usedColumns.add(col.getColumnDesc());
@@ -170,8 +177,15 @@ public class SchemaChecker {
 return violateColumns;
 }
 
-private boolean checkAllColumns(TableDesc table, List fields) 
{
-if (table.getColumnCount() != fields.size()) {
+/**
+ * check whether all columns in `table` are still in `fields` and have the 
same index as before.
+ *
+ * @param table kylin's table metadata
+ * @param fields current table metadata in hive
+ * @return true if only new columns are appended in hive, false otherwise
+ */
+private boolean checkAllColumnsInTableDesc(TableDesc table, 
List fields) {
+if (table.getColumnCount() > fields.size()) {
 return false;
 }
 
@@ -212,15 +226,16 @@ public class SchemaChecker {
 // if user reloads a fact table used by cube, then all used 

[18/50] [abbrv] kylin git commit: KYLIN-2068: add optional comment column when sync hive table columns

[lidong]

KYLIN-2068: add optional comment column when sync hive table columns

Signed-off-by: shaofengshi 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/58afe010
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/58afe010
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/58afe010

Branch: refs/heads/master-hbase1.x
Commit: 58afe0103098dd0845cb50a0751e3305a973f834
Parents: 460536c
Author: Yiming Liu 
Authored: Mon Oct 10 23:49:19 2016 +0800
Committer: shaofengshi 
Committed: Wed Oct 12 09:25:24 2016 +0800

--
 .../apache/kylin/metadata/model/ColumnDesc.java | 13 ++
 examples/sample_cube/create_sample_tables.sql   | 49 +++-
 .../source/hive/HiveSourceTableLoader.java  |  1 +
 webapp/app/js/model/tableConfig.js  |  3 +-
 webapp/app/partials/tables/table_detail.html|  4 ++
 5 files changed, 46 insertions(+), 24 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/58afe010/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
index 257cb3b..d131b92 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
@@ -20,6 +20,7 @@ package org.apache.kylin.metadata.model;
 
 import java.io.Serializable;
 
+import com.fasterxml.jackson.annotation.JsonInclude;
 import org.apache.commons.lang.StringUtils;
 import org.apache.kylin.metadata.datatype.DataType;
 
@@ -51,6 +52,9 @@ public class ColumnDesc implements Serializable {
 private String name;
 @JsonProperty("datatype")
 private String datatype;
+@JsonProperty("comment")
+@JsonInclude(JsonInclude.Include.NON_NULL)
+private String comment;
 
 // parsed from data type
 private DataType type;
@@ -109,6 +113,14 @@ public class ColumnDesc implements Serializable {
 this.table = table;
 }
 
+public String getComment() {
+return comment;
+}
+
+public void setComment(String comment) {
+this.comment = comment;
+}
+
 public DataType getType() {
 return type;
 }
@@ -195,6 +207,7 @@ public class ColumnDesc implements Serializable {
 "id='" + id + '\'' +
 ", name='" + name + '\'' +
 ", datatype='" + datatype + '\'' +
+", comment='" + comment + '\'' +
 '}';
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/58afe010/examples/sample_cube/create_sample_tables.sql
--
diff --git a/examples/sample_cube/create_sample_tables.sql 
b/examples/sample_cube/create_sample_tables.sql
index 8399ee8..fa5e25b 100644
--- a/examples/sample_cube/create_sample_tables.sql
+++ b/examples/sample_cube/create_sample_tables.sql
@@ -20,11 +20,11 @@ DROP TABLE IF EXISTS DEFAULT.KYLIN_CAL_DT;
 
 CREATE TABLE DEFAULT.KYLIN_CAL_DT
 (
-CAL_DT date
-,YEAR_BEG_DT date
-,QTR_BEG_DT date
-,MONTH_BEG_DT date
-,WEEK_BEG_DT date
+CAL_DT date COMMENT 'Date, PK'
+,YEAR_BEG_DT date COMMENT 'YEAR Begin Date'
+,QTR_BEG_DT date COMMENT 'Quarter Begin Date'
+,MONTH_BEG_DT date COMMENT 'Month Begin Date'
+,WEEK_BEG_DT date COMMENT 'Week Begin Date'
 ,AGE_FOR_YEAR_ID smallint
 ,AGE_FOR_QTR_ID smallint
 ,AGE_FOR_MONTH_ID smallint
@@ -121,6 +121,7 @@ CAL_DT date
 ,UPD_DATE string
 ,UPD_USER string
 )
+COMMENT 'Date Dimension Table'
 ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
 STORED AS TEXTFILE;
 
@@ -128,28 +129,28 @@ DROP TABLE IF EXISTS DEFAULT.KYLIN_CATEGORY_GROUPINGS;
 
 CREATE TABLE DEFAULT.KYLIN_CATEGORY_GROUPINGS
 (
-LEAF_CATEG_ID bigint
+LEAF_CATEG_ID bigint COMMENT 'Category ID, PK'
 ,LEAF_CATEG_NAME string
-,SITE_ID int
+,SITE_ID int COMMENT 'Site ID, PK'
 ,CATEG_BUSN_MGR string
 ,CATEG_BUSN_UNIT string
 ,REGN_CATEG string
-,USER_DEFINED_FIELD1 string
-,USER_DEFINED_FIELD3 string
+,USER_DEFINED_FIELD1 string COMMENT 'User Defined Field1'
+,USER_DEFINED_FIELD3 string COMMENT 'User Defined Field3'
 ,CRE_DATE string
-,UPD_DATE string
+,UPD_DATE string COMMENT 'Last Updated Date'
 ,CRE_USER string
-,UPD_USER string
+,UPD_USER string COMMENT 'Last Updated User'
 ,META_CATEG_ID decimal
-,META_CATEG_NAME string
+,META_CATEG_NAME string COMMENT 'Level1 Category'
 ,CATEG_LVL2_ID decimal
 ,CATEG_LVL3_ID decimal
 ,CATEG_LVL4_ID decimal
 ,CATEG_LVL5_ID decimal
 ,CATEG_LVL6_ID decimal
 ,CATEG_LVL7_ID decimal
-,CATEG_LVL2_NAME string
-,CATEG_LVL3_NAME string
+,CATEG_LVL2_NAME string 

[12/50] [abbrv] kylin git commit: KYLIN-2076 Improve sample cube and data

[lidong]

http://git-wip-us.apache.org/repos/asf/kylin/blob/0c7e2688/examples/sample_cube/data/DEFAULT.KYLIN_SALES.csv
--
diff --git a/examples/sample_cube/data/DEFAULT.KYLIN_SALES.csv 
b/examples/sample_cube/data/DEFAULT.KYLIN_SALES.csv
index 13c7d85..dffa43b 100644
--- a/examples/sample_cube/data/DEFAULT.KYLIN_SALES.csv
+++ b/examples/sample_cube/data/DEFAULT.KYLIN_SALES.csv
@@ -1,1 +1,1 @@
-0,2012-12-14,Others,88750,0,11,36.2828,0,1349
-0,2012-08-28,Others,175750,0,13,23.8563,0,1927
-0,2012-02-16,ABIN,148324,15,13,88.3418,0,1005
-0,2013-10-19,FP-non GTC,37831,0,13,47.3015,0,1209
-0,2012-10-22,Others,140746,100,11,83.4540,0,1154
-0,2013-01-25,FP-GTC,16509,0,-99,22.9896,0,1372
-0,2013-04-04,Others,963,0,13,88.5907,0,1648
-0,2012-04-11,Others,15687,0,15,88.1940,0,1866
-0,2013-01-19,ABIN,60606,3,13,77.9727,0,1936
-0,2012-04-30,FP-non GTC,106246,0,14,52.2950,0,1047
-0,2013-02-03,Auction,45333,0,16,56.3584,0,1470
-0,2012-09-02,FP-GTC,25147,0,-99,15.3553,0,1203
-0,2013-07-30,Others,24760,0,16,25.0770,0,1382
-0,2012-04-17,Auction,31519,0,16,72.7384,0,1069
-0,2013-04-30,FP-GTC,51582,0,15,75.8200,0,1085
-0,2013-03-23,FP-GTC,16509,0,15,8.6653,0,1559
-0,2013-04-01,ABIN,175750,3,16,64.2802,0,1507
-0,2013-07-12,ABIN,50677,0,-99,24.5987,0,1938
-0,2012-04-23,FP-GTC,45333,0,5,72.6553,0,1863
-0,2012-12-19,ABIN,3838,0,12,87.4142,0,1612
-0,2012-08-15,FP-GTC,62179,0,16,67.4238,0,1839
-0,2013-08-22,Auction,1357,0,5,18.3600,0,1303
-0,2013-08-23,ABIN,31519,0,5,67.6344,0,1877
-0,2013-09-12,FP-GTC,223,0,14,71.7898,0,1848
-0,2012-03-14,FP-GTC,132939,0,15,26.1496,0,1815
-0,2013-07-13,Others,67703,3,16,83.9661,0,1858
-0,2013-12-15,FP-non GTC,100847,0,5,34.3451,0,1387
-0,2012-08-30,Auction,62179,0,11,21.0496,0,1393
-0,2013-01-10,FP-GTC,48027,0,13,23.4142,0,1757
-0,2013-01-25,ABIN,145970,0,5,26.7842,0,1402
-0,2012-07-05,Auction,48027,0,-99,46.7320,0,1972
-0,2012-11-07,FP-GTC,20213,0,-99,19.7270,0,1873
-0,2013-06-20,ABIN,100847,0,5,76.2783,0,1409
-0,2012-10-08,ABIN,26262,0,5,48.7447,0,1029
-0,2013-04-30,ABIN,87118,0,15,63.2187,0,1529
-0,2013-04-24,FP-non GTC,20213,0,16,39.5057,0,1329
-0,2012-05-13,Others,164261,0,13,84.1246,0,1688
-0,2013-08-02,ABIN,31519,0,5,37.1504,0,1905
-0,2012-04-03,Auction,164261,0,11,92.0974,0,1341
-0,2012-09-02,FP-GTC,31519,0,14,87.7829,0,1460
-0,2013-10-05,ABIN,13987,0,16,74.0719,0,1381
-0,2012-01-13,FP-GTC,48027,0,15,71.2637,0,1915
-0,2013-03-01,Others,13836,0,14,16.7288,0,1953
-0,2012-09-12,FP-non GTC,16509,0,5,12.2933,0,1277
-0,2012-02-07,Auction,45333,0,5,64.9770,0,1040
-0,2013-12-14,FP-non GTC,158798,0,16,72.4413,0,1500
-0,2013-10-13,Auction,31519,0,5,79.3053,0,1816
-0,2012-06-11,ABIN,4943,0,5,11.6942,0,1696
-0,2012-10-18,ABIN,80053,0,-99,54.0933,0,1604
-0,2012-11-03,Auction,20886,0,5,9.8258,0,1764
-0,2012-01-15,Auction,44079,0,14,13.0371,0,1343
-0,2012-07-03,FP-GTC,159184,0,-99,92.5314,0,1724
-0,2012-02-16,FP-GTC,314,0,13,49.1825,0,1334
-0,2012-06-17,FP-non GTC,31387,3,12,0.6677,0,1603
-0,2012-07-15,FP-non GTC,32876,0,-99,50.9634,0,1248
-0,2012-07-24,FP-GTC,150265,15,16,57.5645,0,1653
-0,2012-10-06,Auction,145970,0,5,44.3091,0,1082
-0,2013-06-14,ABIN,80053,0,5,20.2388,0,1632
-0,2013-10-10,FP-non GTC,15868,0,11,28.6290,0,1523
-0,2012-09-30,ABIN,31673,0,12,75.8658,0,1719
-0,2012-04-02,FP-GTC,63861,0,5,25.8748,0,1311
-0,2012-03-16,Others,314,0,-99,9.2733,0,1409
-0,2012-09-30,FP-non GTC,60340,0,14,74.6610,0,1790
-0,2012-03-20,Others,87118,0,5,79.3781,0,1384
-0,2012-09-12,FP-GTC,1120,3,5,61.4408,0,1645
-0,2013-12-10,FP-non GTC,20886,0,11,14.8140,0,1943
-0,2012-12-05,FP-GTC,106246,0,12,24.9380,0,1858
-0,2012-08-21,Auction,43479,0,12,22.2008,0,1310
-0,2012-11-30,FP-GTC,175750,0,15,35.1161,0,1440
-0,2013-10-10,Others,95672,0,11,82.3718,0,1128
-0,2012-10-17,Auction,80053,0,15,9.8166,0,1105
-0,2012-03-16,FP-non GTC,57784,0,5,66.2369,0,1439
-0,2012-11-14,ABIN,65,0,11,17.5513,0,1855
-0,2013-06-24,FP-GTC,16509,0,15,53.9195,0,1130
-0,2012-08-13,FP-non GTC,65,0,5,69.8447,0,1548
-0,2012-01-03,Others,38238,0,11,90.3722,0,1739
-0,2013-11-24,FP-non GTC,175750,3,13,47.3441,0,1291
-0,2013-11-08,Auction,106246,0,5,38.6729,0,1310
-0,2013-04-07,Others,73506,0,-99,69.8999,0,1235
-0,2012-10-15,FP-GTC,314,0,13,44.1598,0,1265
-0,2012-01-20,FP-GTC,80287,0,-99,85.7263,0,1704
-0,2012-08-17,ABIN,32876,0,16,21.2934,0,1163
-0,2013-07-09,Auction,37831,0,15,38.2213,0,1557
-0,2013-09-21,Others,45333,0,13,25.5384,0,1184
-0,2013-10-28,FP-non GTC,11554,0,16,89.9654,0,1940
-0,2013-04-05,Auction,11554,0,14,41.8676,0,1384
-0,2013-03-25,FP-GTC,43398,0,14,75.0110,0,1697
-0,2012-05-03,ABIN,33038,15,12,64.6735,0,1147

kylin git commit: KYLIN-2105 add QueryId

[gaodayue]

Repository: kylin
Updated Branches:
  refs/heads/master 6cafc422b -> db09f5f9c


KYLIN-2105 add QueryId


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/db09f5f9
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/db09f5f9
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/db09f5f9

Branch: refs/heads/master
Commit: db09f5f9cae5a3d3ff731221cbb1c026da4f4e41
Parents: 6cafc42
Author: gaodayue 
Authored: Wed Oct 19 08:36:53 2016 +0800
Committer: gaodayue 
Committed: Wed Oct 19 08:36:53 2016 +0800

--
 .../kylin/common/debug/BackdoorToggles.java |   6 +
 .../apache/kylin/common/util/SetThreadName.java |  49 +
 .../apache/kylin/rest/service/QueryService.java |  32 ++-
 .../kylin/rest/util/QueryIdGenerator.java   |  45 +
 .../kylin/rest/util/QueryIdGeneratorTest.java   | 106 ++
 .../src/main/resources/applicationContext.xml   |   1 +
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java |   7 +-
 .../coprocessor/endpoint/CubeVisitService.java  |   4 +-
 .../endpoint/generated/CubeVisitProtos.java | 197 +--
 .../endpoint/protobuf/CubeVisit.proto   |   1 +
 10 files changed, 420 insertions(+), 28 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/db09f5f9/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java 
b/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
index f88781a..f7c90aa 100644
--- 
a/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
+++ 
b/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
@@ -65,6 +65,10 @@ public class BackdoorToggles {
 return Integer.valueOf(v);
 }
 
+public static String getQueryId() {
+return getString(KEY_QUERY_ID);
+}
+
 public static Pair getShardAssignment() {
 String v = getString(DEBUG_TOGGLE_SHARD_ASSIGNMENT);
 if (v == null) {
@@ -92,6 +96,8 @@ public class BackdoorToggles {
 _backdoorToggles.remove();
 }
 
+public final static String KEY_QUERY_ID = "QUERY_ID";
+
 /**
  * set DEBUG_TOGGLE_DISABLE_FUZZY_KEY=true to disable fuzzy key for 
debug/profile usage
  *

http://git-wip-us.apache.org/repos/asf/kylin/blob/db09f5f9/core-common/src/main/java/org/apache/kylin/common/util/SetThreadName.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/util/SetThreadName.java 
b/core-common/src/main/java/org/apache/kylin/common/util/SetThreadName.java
new file mode 100644
index 000..9e5197e
--- /dev/null
+++ b/core-common/src/main/java/org/apache/kylin/common/util/SetThreadName.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.common.util;
+
+import java.io.Closeable;
+
+/**
+ *
+ * An utility that makes it easy to change current thread name in a 
try-with-resource block.
+ *
+ *  Example:
+ *
+ * 
+ * // old thread name
+ * try (SetThreadName ignored = new SetThreadName(format, arg1, arg2)) {
+ * // new thread name
+ * }
+ * // old thread name
+ * 
+ */
+public class SetThreadName implements Closeable {
+private final String originThreadName;
+
+public SetThreadName(String format, Object... args) {
+originThreadName = Thread.currentThread().getName();
+Thread.currentThread().setName(String.format(format, args) + "-" + 
Thread.currentThread().getId());
+}
+
+@Override
+public void close() {
+Thread.currentThread().setName(originThreadName);
+}
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/db09f5f9/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
--

kylin git commit: KYLIN-1839 minor refactor

[liyang]

Repository: kylin
Updated Branches:
  refs/heads/master aa1550c9e -> 6cafc422b


KYLIN-1839 minor refactor


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6cafc422
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6cafc422
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6cafc422

Branch: refs/heads/master
Commit: 6cafc422bc0ca2fa19a4516bcb4388719c99184c
Parents: aa1550c
Author: Yang Li 
Authored: Wed Oct 19 08:19:35 2016 +0800
Committer: Yang Li 
Committed: Wed Oct 19 08:30:49 2016 +0800

--
 .../kylin/engine/mr/common/AbstractHadoopJob.java  | 13 +++--
 1 file changed, 3 insertions(+), 10 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/6cafc422/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
index f70e3bb..a5b2d2e 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
@@ -318,7 +318,7 @@ public abstract class AbstractHadoopJob extends Configured 
implements Tool {
 }
 
 if (fs.getFileStatus(p).isDirectory()) {
-appendTmpDir(job, fs, p);
+appendTmpDir(job, fs, p, jarList, fileList);
 continue;
 }
 
@@ -335,18 +335,14 @@ public abstract class AbstractHadoopJob extends 
Configured implements Tool {
 }
 }
 
-private void appendTmpDir(Job job, FileSystem fs, Path tmpDir) {
+private void appendTmpDir(Job job, FileSystem fs, Path tmpDir, 
StringBuilder jarList, StringBuilder fileList) {
 try {
-Configuration jobConf = job.getConfiguration();
 FileStatus[] fList = fs.listStatus(tmpDir);
 
-StringBuilder jarList = new StringBuilder();
-StringBuilder fileList = new StringBuilder();
-
 for (FileStatus file : fList) {
 Path p = file.getPath();
 if (fs.getFileStatus(p).isDirectory()) {
-appendTmpDir(job, fs, p);
+appendTmpDir(job, fs, p, jarList, fileList);
 continue;
 }
 
@@ -356,9 +352,6 @@ public abstract class AbstractHadoopJob extends Configured 
implements Tool {
 list.append(fs.getFileStatus(p).getPath().toString());
 }
 
-appendTmpFiles(fileList.toString(), jobConf);
-appendTmpJars(jarList.toString(), jobConf);
-
 } catch (IOException e) {
 throw new RuntimeException(e);
 }

[1/2] kylin git commit: KYLIN-1839, support kylin lib in HDFS

[liyang]

Repository: kylin
Updated Branches:
  refs/heads/master d28835f69 -> aa1550c9e


KYLIN-1839, support kylin lib in HDFS

Signed-off-by: terry 
Signed-off-by: Yang Li 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4473d710
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4473d710
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4473d710

Branch: refs/heads/master
Commit: 4473d71011cc0e652eccf4f80269828caa5d3c73
Parents: d28835f
Author: terry 
Authored: Tue Oct 11 17:33:45 2016 +0800
Committer: Yang Li 
Committed: Wed Oct 19 07:30:13 2016 +0800

--
 .../engine/mr/common/AbstractHadoopJob.java | 29 +++-
 1 file changed, 16 insertions(+), 13 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/4473d710/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
index a138eec..bbb1711 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
@@ -46,6 +46,7 @@ import org.apache.hadoop.conf.Configured;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
 import org.apache.hadoop.mapreduce.Counters;
 import org.apache.hadoop.mapreduce.InputFormat;
 import org.apache.hadoop.mapreduce.InputSplit;
@@ -252,14 +253,10 @@ public abstract class AbstractHadoopJob extends 
Configured implements Tool {
 // for KylinJobMRLibDir
 String mrLibDir = kylinConf.getKylinJobMRLibDir();
 if (!StringUtils.isBlank(mrLibDir)) {
-File dirFileMRLIB = new File(mrLibDir);
-if (dirFileMRLIB.exists()) {
-if (kylinDependency.length() > 0)
-kylinDependency.append(",");
+if(kylinDependency.length() > 0) {
+kylinDependency.append(",");
+}
 kylinDependency.append(mrLibDir);
-} else {
-logger.info("The directory '" + mrLibDir + "' for 
'kylin.job.mr.lib.dir' does not exist!!!");
-}
 }
 
 setJobTmpJarsAndFiles(job, kylinDependency.toString());
@@ -300,21 +297,27 @@ public abstract class AbstractHadoopJob extends 
Configured implements Tool {
 try {
 Configuration jobConf = job.getConfiguration();
 FileSystem fs = FileSystem.getLocal(jobConf);
+FileSystem hdfs = FileSystem.get(jobConf);
 
 StringBuilder jarList = new StringBuilder();
 StringBuilder fileList = new StringBuilder();
 
 for (String fileName : fNameList) {
 Path p = new Path(fileName);
-if (fs.getFileStatus(p).isDirectory()) {
-appendTmpDir(job, fileName);
+FileSystem current = 
(fileName.startsWith(HdfsConstants.HDFS_URI_SCHEME) ? hdfs : fs);
+if(!current.exists(p)) {
+logger.warn("The directory '" + fileName + "for kylin 
dependency does not exist!!!");
+continue;
+}
+if (current.getFileStatus(p).isDirectory()) {
+appendTmpDir(job, current, fileName);
 continue;
 }
 
 StringBuilder list = (p.getName().endsWith(".jar")) ? jarList 
: fileList;
 if (list.length() > 0)
 list.append(",");
-list.append(fs.getFileStatus(p).getPath().toString());
+list.append(current.getFileStatus(p).getPath());
 }
 
 appendTmpFiles(fileList.toString(), jobConf);
@@ -324,13 +327,12 @@ public abstract class AbstractHadoopJob extends 
Configured implements Tool {
 }
 }
 
-private void appendTmpDir(Job job, String tmpDir) {
+private void appendTmpDir(Job job, FileSystem fs, String tmpDir) {
 if (StringUtils.isBlank(tmpDir))
 return;
 
 try {
 Configuration jobConf = job.getConfiguration();
-FileSystem fs = FileSystem.getLocal(jobConf);
 FileStatus[] fList = fs.listStatus(new Path(tmpDir));
 
 StringBuilder jarList = new StringBuilder();
@@ -339,7 +341,7 @@ public abstract class AbstractHadoopJob extends Configured 
implements Tool {
 for (FileStatus file 

[2/2] kylin git commit: KYLIN-1839 code review

[liyang]

KYLIN-1839 code review


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/aa1550c9
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/aa1550c9
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/aa1550c9

Branch: refs/heads/master
Commit: aa1550c9ecfb1c047c107fa51187a51e958ab189
Parents: 4473d71
Author: Yang Li 
Authored: Wed Oct 19 08:19:35 2016 +0800
Committer: Yang Li 
Committed: Wed Oct 19 08:19:35 2016 +0800

--
 .../engine/mr/common/AbstractHadoopJob.java | 44 +++-
 1 file changed, 24 insertions(+), 20 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/aa1550c9/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
index bbb1711..f70e3bb 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
@@ -46,7 +46,6 @@ import org.apache.hadoop.conf.Configured;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.protocol.HdfsConstants;
 import org.apache.hadoop.mapreduce.Counters;
 import org.apache.hadoop.mapreduce.InputFormat;
 import org.apache.hadoop.mapreduce.InputSplit;
@@ -80,7 +79,7 @@ public abstract class AbstractHadoopJob extends Configured 
implements Tool {
 protected static final Option OPTION_JOB_NAME = 
OptionBuilder.withArgName(BatchConstants.ARG_JOB_NAME).hasArg().isRequired(true).withDescription("Job
 name. For example, 
Kylin_Cuboid_Builder-clsfd_v2_Step_22-D)").create(BatchConstants.ARG_JOB_NAME);
 protected static final Option OPTION_CUBE_NAME = 
OptionBuilder.withArgName(BatchConstants.ARG_CUBE_NAME).hasArg().isRequired(true).withDescription("Cube
 name. For exmaple, flat_item_cube").create(BatchConstants.ARG_CUBE_NAME);
 protected static final Option OPTION_CUBING_JOB_ID = 
OptionBuilder.withArgName(BatchConstants.ARG_CUBING_JOB_ID).hasArg().isRequired(false).withDescription("ID
 of cubing job executable").create(BatchConstants.ARG_CUBING_JOB_ID);
-//@Deprecated
+//@Deprecated
 protected static final Option OPTION_SEGMENT_NAME = 
OptionBuilder.withArgName(BatchConstants.ARG_SEGMENT_NAME).hasArg().isRequired(true).withDescription("Cube
 segment name").create(BatchConstants.ARG_SEGMENT_NAME);
 protected static final Option OPTION_SEGMENT_ID = 
OptionBuilder.withArgName(BatchConstants.ARG_SEGMENT_ID).hasArg().isRequired(true).withDescription("Cube
 segment id").create(BatchConstants.ARG_SEGMENT_ID);
 protected static final Option OPTION_INPUT_PATH = 
OptionBuilder.withArgName(BatchConstants.ARG_INPUT).hasArg().isRequired(true).withDescription("Input
 path").create(BatchConstants.ARG_INPUT);
@@ -253,10 +252,10 @@ public abstract class AbstractHadoopJob extends 
Configured implements Tool {
 // for KylinJobMRLibDir
 String mrLibDir = kylinConf.getKylinJobMRLibDir();
 if (!StringUtils.isBlank(mrLibDir)) {
-if(kylinDependency.length() > 0) {
-kylinDependency.append(",");
-}
-kylinDependency.append(mrLibDir);
+if (kylinDependency.length() > 0) {
+kylinDependency.append(",");
+}
+kylinDependency.append(mrLibDir);
 }
 
 setJobTmpJarsAndFiles(job, kylinDependency.toString());
@@ -296,7 +295,7 @@ public abstract class AbstractHadoopJob extends Configured 
implements Tool {
 
 try {
 Configuration jobConf = job.getConfiguration();
-FileSystem fs = FileSystem.getLocal(jobConf);
+FileSystem localfs = FileSystem.getLocal(jobConf);
 FileSystem hdfs = FileSystem.get(jobConf);
 
 StringBuilder jarList = new StringBuilder();
@@ -304,20 +303,29 @@ public abstract class AbstractHadoopJob extends 
Configured implements Tool {
 
 for (String fileName : fNameList) {
 Path p = new Path(fileName);
-FileSystem current = 
(fileName.startsWith(HdfsConstants.HDFS_URI_SCHEME) ? hdfs : fs);
-if(!current.exists(p)) {
-logger.warn("The directory '" + fileName + "for kylin 
dependency does not exist!!!");
+if (p.isAbsolute() == false) {
+logger.warn("The directory of kylin dependency '" + 
fileName + "' is not absolute, skip");
+continue;
+   

[2/4] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[liyang]

http://git-wip-us.apache.org/repos/asf/kylin/blob/503fd4d1/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-if (admin.isTableEnabled(table)) {
-admin.disableTable(table);
+if 
(admin.isTableEnabled(TableName.valueOf(table))) {
+admin.disableTable(TableName.valueOf(table));
 }
-admin.deleteTable(table);
+admin.deleteTable(TableName.valueOf(table));
 logger.debug("Dropped htable: " + table);
 output.append("HBase table " + table + " is 
dropped. \n");
 } else {

http://git-wip-us.apache.org/repos/asf/kylin/blob/503fd4d1/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index a150607..56f867a 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,9 +21,11 @@ package org.apache.kylin.storage.hbase.util;
 import java.io.IOException;
 
 import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.AbstractApplication;
 import org.apache.kylin.common.util.OptionsHelper;
 import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -38,8 +40,8 @@ public class CleanHtableCLI extends AbstractApplication {
 protected static final Logger logger = 
LoggerFactory.getLogger(CleanHtableCLI.class);
 
 private void clean() throws IOException {
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();
 
 for (HTableDescriptor descriptor : hbaseAdmin.listTables()) {
 String name = 

[4/4] kylin git commit: KYLIN-1672 support kylin on cdh 5.7

[liyang]

KYLIN-1672 support kylin on cdh 5.7

Signed-off-by: Li Yang 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/97201875
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/97201875
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/97201875

Branch: refs/heads/yang21-cdh5.7
Commit: 97201875f69730c6fbf3bda73343d2ada137093e
Parents: 503fd4d
Author: Lynne Jiang 
Authored: Mon May 16 03:33:27 2016 -0700
Committer: Yang Li 
Committed: Tue Oct 18 22:31:54 2016 +

--
 dev-support/test_all_against_hdp_2_2_4_2_2.sh   |   0
 .../kylin/engine/mr/steps/MockupMapContext.java |  15 +-
 examples/test_case_data/sandbox/core-site.xml   | 146 +++---
 examples/test_case_data/sandbox/hbase-site.xml  | 162 ++
 examples/test_case_data/sandbox/hdfs-site.xml   | 259 ++
 examples/test_case_data/sandbox/mapred-site.xml | 398 ++-
 examples/test_case_data/sandbox/yarn-site.xml   | 496 ++-
 pom.xml |  16 +-
 server/pom.xml  |  36 ++
 .../storage/hbase/steps/MockupMapContext.java   |  19 +-
 tool/pom.xml|  12 +
 11 files changed, 428 insertions(+), 1131 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/97201875/dev-support/test_all_against_hdp_2_2_4_2_2.sh
--
diff --git a/dev-support/test_all_against_hdp_2_2_4_2_2.sh 
b/dev-support/test_all_against_hdp_2_2_4_2_2.sh
old mode 100644
new mode 100755

http://git-wip-us.apache.org/repos/asf/kylin/blob/97201875/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
--
diff --git 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
index 847071d..9900465 100644
--- 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
+++ 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
@@ -77,6 +77,7 @@ public class MockupMapContext {
 outKV[0] = key;
 outKV[1] = value;
 }
+
 }
 
 @Override
@@ -99,6 +100,7 @@ public class MockupMapContext {
 throw new NotImplementedException();
 }
 
+
 @Override
 public float getProgress() {
 throw new NotImplementedException();
@@ -195,17 +197,17 @@ public class MockupMapContext {
 }
 
 @Override
-public RawComparator getSortComparator() {
+public boolean userClassesTakesPrecedence() {
 throw new NotImplementedException();
 }
 
 @Override
-public String getJar() {
+public RawComparator getSortComparator() {
 throw new NotImplementedException();
 }
 
 @Override
-public RawComparator getGroupingComparator() {
+public String getJar() {
 throw new NotImplementedException();
 }
 
@@ -221,7 +223,7 @@ public class MockupMapContext {
 
 @Override
 public boolean getProfileEnabled() {
-throw new NotImplementedException();
+return false;
 }
 
 @Override
@@ -308,6 +310,11 @@ public class MockupMapContext {
 public RawComparator getCombinerKeyGroupingComparator() {
 throw new NotImplementedException();
 }
+
+@Override
+public RawComparator getGroupingComparator() {
+return null;
+}
 });
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/97201875/examples/test_case_data/sandbox/core-site.xml
--
diff --git a/examples/test_case_data/sandbox/core-site.xml 
b/examples/test_case_data/sandbox/core-site.xml
index 9aa588c..6162406 100644
--- a/examples/test_case_data/sandbox/core-site.xml
+++ b/examples/test_case_data/sandbox/core-site.xml
@@ -14,152 +14,146 @@
   See the License for the specific language governing permissions and
   limitations under the License.
 -->
+
 
-
 
 fs.defaultFS
-hdfs://sandbox.hortonworks.com:8020
-true
+hdfs://quickstart.cloudera:8020
 
-
 
 fs.trash.interval
-360
+1
 
-
 
-
ha.failover-controller.active-standby-elector.zk.op.retries
-120
+io.compression.codecs
+

[1/4] kylin git commit: KYLIN-2099 fix sample model [Forced Update!]

[liyang]

Repository: kylin
Updated Branches:
  refs/heads/yang21-cdh5.7 41a19e917 -> 97201875f (forced update)


KYLIN-2099 fix sample model


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/274d5ce8
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/274d5ce8
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/274d5ce8

Branch: refs/heads/yang21-cdh5.7
Commit: 274d5ce8f640d2d3a0e3ddd9c75a505748843b3e
Parents: 8006bb1
Author: Li Yang 
Authored: Tue Oct 18 12:36:43 2016 +0800
Committer: Li Yang 
Committed: Tue Oct 18 12:36:43 2016 +0800

--
 examples/sample_cube/template/model_desc/kylin_sales_model.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/274d5ce8/examples/sample_cube/template/model_desc/kylin_sales_model.json
--
diff --git a/examples/sample_cube/template/model_desc/kylin_sales_model.json 
b/examples/sample_cube/template/model_desc/kylin_sales_model.json
index 13fc356..3c4fa5a 100644
--- a/examples/sample_cube/template/model_desc/kylin_sales_model.json
+++ b/examples/sample_cube/template/model_desc/kylin_sales_model.json
@@ -21,7 +21,7 @@
 "columns" : ["CAL_DT", "WEEK_BEG_DT"]
   }, {
 "table" : "DEFAULT.KYLIN_CATEGORY_GROUPINGS",
-"columns" : [ "USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3", "UPD_DATE", 
"UPD_USER", "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME" ]
+"columns" : [ "USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3", 
"META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME" ]
   }, {
 "table" : "DEFAULT.KYLIN_SALES",
 "columns" : [ "LSTG_FORMAT_NAME", "SELLER_ID", "PART_DT", "USER_ID", 
"REGION" ]

[3/4] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[liyang]

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/503fd4d1
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/503fd4d1
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/503fd4d1

Branch: refs/heads/yang21-cdh5.7
Commit: 503fd4d12ecfb2bef5dfeb50be657b73d37e4485
Parents: 274d5ce
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: Yang Li 
Committed: Tue Oct 18 22:29:37 2016 +

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 13 ++-
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 36 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  6 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 23 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../hbase/steps/HBaseStreamingOutput.java   |  9 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 36 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 22 ++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 49 files changed, 408 insertions(+), 462 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/503fd4d1/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-etc.  Retry interval is a rough function based on 
hbase.client.pause.  At
-first we retry at this interval but then with backoff, we pretty 
quickly reach
-retrying every ten seconds.  See 

[1/3] kylin git commit: KYLIN-2099 fix sample model [Forced Update!]

[liyang]

Repository: kylin
Updated Branches:
  refs/heads/yang21-hbase1.x a27ff16ed -> 503fd4d12 (forced update)


KYLIN-2099 fix sample model


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/274d5ce8
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/274d5ce8
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/274d5ce8

Branch: refs/heads/yang21-hbase1.x
Commit: 274d5ce8f640d2d3a0e3ddd9c75a505748843b3e
Parents: 8006bb1
Author: Li Yang 
Authored: Tue Oct 18 12:36:43 2016 +0800
Committer: Li Yang 
Committed: Tue Oct 18 12:36:43 2016 +0800

--
 examples/sample_cube/template/model_desc/kylin_sales_model.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/274d5ce8/examples/sample_cube/template/model_desc/kylin_sales_model.json
--
diff --git a/examples/sample_cube/template/model_desc/kylin_sales_model.json 
b/examples/sample_cube/template/model_desc/kylin_sales_model.json
index 13fc356..3c4fa5a 100644
--- a/examples/sample_cube/template/model_desc/kylin_sales_model.json
+++ b/examples/sample_cube/template/model_desc/kylin_sales_model.json
@@ -21,7 +21,7 @@
 "columns" : ["CAL_DT", "WEEK_BEG_DT"]
   }, {
 "table" : "DEFAULT.KYLIN_CATEGORY_GROUPINGS",
-"columns" : [ "USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3", "UPD_DATE", 
"UPD_USER", "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME" ]
+"columns" : [ "USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3", 
"META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME" ]
   }, {
 "table" : "DEFAULT.KYLIN_SALES",
 "columns" : [ "LSTG_FORMAT_NAME", "SELLER_ID", "PART_DT", "USER_ID", 
"REGION" ]

[2/3] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[liyang]

http://git-wip-us.apache.org/repos/asf/kylin/blob/503fd4d1/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-if (admin.isTableEnabled(table)) {
-admin.disableTable(table);
+if 
(admin.isTableEnabled(TableName.valueOf(table))) {
+admin.disableTable(TableName.valueOf(table));
 }
-admin.deleteTable(table);
+admin.deleteTable(TableName.valueOf(table));
 logger.debug("Dropped htable: " + table);
 output.append("HBase table " + table + " is 
dropped. \n");
 } else {

http://git-wip-us.apache.org/repos/asf/kylin/blob/503fd4d1/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index a150607..56f867a 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,9 +21,11 @@ package org.apache.kylin.storage.hbase.util;
 import java.io.IOException;
 
 import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.AbstractApplication;
 import org.apache.kylin.common.util.OptionsHelper;
 import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -38,8 +40,8 @@ public class CleanHtableCLI extends AbstractApplication {
 protected static final Logger logger = 
LoggerFactory.getLogger(CleanHtableCLI.class);
 
 private void clean() throws IOException {
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();
 
 for (HTableDescriptor descriptor : hbaseAdmin.listTables()) {
 String name = 

[3/3] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[liyang]

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/503fd4d1
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/503fd4d1
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/503fd4d1

Branch: refs/heads/yang21-hbase1.x
Commit: 503fd4d12ecfb2bef5dfeb50be657b73d37e4485
Parents: 274d5ce
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: Yang Li 
Committed: Tue Oct 18 22:29:37 2016 +

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 13 ++-
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 36 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  6 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 23 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../hbase/steps/HBaseStreamingOutput.java   |  9 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 36 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 22 ++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 49 files changed, 408 insertions(+), 462 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/503fd4d1/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-etc.  Retry interval is a rough function based on 
hbase.client.pause.  At
-first we retry at this interval but then with backoff, we pretty 
quickly reach
-retrying every ten seconds.  See 

kylin git commit: minor update on the blog

[shaofengshi]

Repository: kylin
Updated Branches:
  refs/heads/document 38cd798d0 -> 6e3a4bc00


minor update on the blog


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6e3a4bc0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6e3a4bc0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6e3a4bc0

Branch: refs/heads/document
Commit: 6e3a4bc004ca2bbb050d3851cdac4b5a21794dd4
Parents: 38cd798
Author: shaofengshi 
Authored: Tue Oct 18 22:25:37 2016 +0800
Committer: shaofengshi 
Committed: Tue Oct 18 22:25:37 2016 +0800

--
 .../_posts/blog/2016-10-18-new-nrt-streaming.md   | 18 ++
 1 file changed, 10 insertions(+), 8 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/6e3a4bc0/website/_posts/blog/2016-10-18-new-nrt-streaming.md
--
diff --git a/website/_posts/blog/2016-10-18-new-nrt-streaming.md 
b/website/_posts/blog/2016-10-18-new-nrt-streaming.md
index e1cb845..ad75636 100644
--- a/website/_posts/blog/2016-10-18-new-nrt-streaming.md
+++ b/website/_posts/blog/2016-10-18-new-nrt-streaming.md
@@ -18,16 +18,16 @@ While, that implementation was marked as "experimental" 
because it has the follo
 
  * Others: hard to recover from accident, difficult to maintain the code, etc.
 
-To overcome these limitations, the Apache Kylin team developed the new 
streaming ([KYLIN-1726](https://issues.apache.org/jira/browse/KYLIN-1726)) with 
Kafka 0.10 API, it has been tested internally for some time, will release to 
public soon.
+To overcome these limitations, the Apache Kylin team developed the new 
streaming ([KYLIN-1726](https://issues.apache.org/jira/browse/KYLIN-1726)) with 
Kafka 0.10, it has been tested internally for some time, will release to public 
soon.
 
-The new design is a perfect implementation under Kylin 1.5's "Plug-in" 
architecture: treat Kafka topic as a "Data Source" like Hive table, using an 
adapter to extract the data to HDFS; the next steps are almost the same as from 
Hive. Figure 1 is a high level architecture of the new design.
+The new design is a perfect implementation under Kylin 1.5's "plug-in" 
architecture: treat Kafka topic as a "Data Source" like Hive table, using an 
adapter to extract the data to HDFS; the next steps are almost the same as 
other cubes. Figure 1 is a high level architecture of the new design.
 
 
 ![Kylin New Streaming Framework Architecture](/images/blog/new-streaming.png)
 
-The adapter to read Kafka messages is modified from 
[kafka-hadoop-loader](https://github.com/amient/kafka-hadoop-loader), which is 
open sourced under Apache License V2.0; it starts a mapper for each Kafka 
partition, reading and then saving the messages to HDFS; in next steps Kylin 
will be able to leverage existing framework like MR to do the processing, this 
makes the solution scalable and fault-tolerant. 
+The adapter to read Kafka messages is modified from 
[kafka-hadoop-loader](https://github.com/amient/kafka-hadoop-loader), the 
author Michal Harish open sourced it under Apache License V2.0; it starts a 
mapper for each Kafka partition, reading and then saving the messages to HDFS; 
so Kylin will be able to leverage existing framework like MR to do the 
processing, this makes the solution scalable and fault-tolerant. 
 
-To overcome the "data loss" problem, Kylin adds the start/end offset 
information on each Cube segment, and then use the offsets as the partition 
value (no overlap is allowed); this ensures no data be lost and 1 message be 
consumed at most once. To let the late/early message can be queried, Cube 
segments allow overlap for the partition time dimension: Kylin will scan all 
segments which include the queried time. Figure 2 illurates this.
+To overcome the "data loss" limitation, Kylin adds the start/end offset 
information on each Cube segment, and then use the offsets as the partition 
value (no overlap allowed); this ensures no data be lost and 1 message be 
consumed at most once. To let the late/early message can be queried, Cube 
segments allow overlap for the partition time dimension: each segment has a 
"min" date/time and a "max" date/time; Kylin will scan all segments which 
matched with the queried time scope. Figure 2 illurates this.
 
 ![Use Offset to Cut Segments](/images/blog/offset-as-partition-value.png)
 
@@ -39,18 +39,20 @@ Other changes/enhancements are made in the new streaming:
  * Add REST API to trigger streaming cube's building
  * Add REST API to check and fill the segment holes
 
-The integration test result shows big improvements than the previous version:
+The integration test result is promising:
 
  * Scalability: it can easily process up to hundreds of million records in one 
build; 
- * 

kylin git commit: minor, clear MassIn unused code

[liyang]

Repository: kylin
Updated Branches:
  refs/heads/master c84892f52 -> d28835f69


minor, clear MassIn unused code

Signed-off-by: Li Yang 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d28835f6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d28835f6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d28835f6

Branch: refs/heads/master
Commit: d28835f69b088d8b08ed0d1bb6514116e280fd13
Parents: c84892f
Author: Roger Shi 
Authored: Tue Oct 18 15:42:10 2016 +0800
Committer: Li Yang 
Committed: Tue Oct 18 16:29:43 2016 +0800

--
 .../metadata/filter/UDF/MassInTupleFilter.java  |   9 +-
 .../apache/kylin/query/ITMassInQueryTest.java   |   3 +
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  12 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  11 --
 .../filter/MassInValueProviderFactoryImpl.java  |  43 ---
 .../cube/v2/filter/MassInValueProviderImpl.java | 117 ---
 6 files changed, 12 insertions(+), 183 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/d28835f6/core-metadata/src/main/java/org/apache/kylin/metadata/filter/UDF/MassInTupleFilter.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/UDF/MassInTupleFilter.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/UDF/MassInTupleFilter.java
index 66c9f21..3f0546c 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/UDF/MassInTupleFilter.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/UDF/MassInTupleFilter.java
@@ -71,7 +71,14 @@ public class MassInTupleFilter extends FunctionTupleFilter {
 
 @Override
 public Collection getValues() {
-return null;
+if (valueProvider == null) {
+valueProvider = 
VALUE_PROVIDER_FACTORY.getProvider(filterTableType, 
filterTableResourceIdentifier, column);
+}
+return valueProvider.getMassInValues();
+}
+
+public TblColRef getColumn() {
+return column;
 }
 
 @Override

http://git-wip-us.apache.org/repos/asf/kylin/blob/d28835f6/kylin-it/src/test/java/org/apache/kylin/query/ITMassInQueryTest.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/query/ITMassInQueryTest.java 
b/kylin-it/src/test/java/org/apache/kylin/query/ITMassInQueryTest.java
index 1831dd9..5835baf 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/ITMassInQueryTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/ITMassInQueryTest.java
@@ -41,6 +41,7 @@ import org.junit.After;
 import org.junit.AfterClass;
 import org.junit.Before;
 import org.junit.BeforeClass;
+import org.junit.Ignore;
 import org.junit.Test;
 
 import com.google.common.collect.Sets;
@@ -92,11 +93,13 @@ public class ITMassInQueryTest extends KylinTestBase {
 }
 
 @Test
+@Ignore
 public void testMassInQuery() throws Exception {
 compare("src/test/resources/query/sql_massin", null, true);
 }
 
 @Test
+@Ignore
 public void testMassInWithDistinctCount() throws Exception {
 run("src/test/resources/query/sql_massin_distinct", null, true);
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/d28835f6/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
index 5692000..3cefc5f 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
@@ -32,19 +32,15 @@ import org.apache.hadoop.hbase.client.Scan;
 import org.apache.kylin.common.util.BytesUtil;
 import org.apache.kylin.common.util.ImmutableBitSet;
 import org.apache.kylin.common.util.ShardingHash;
-import org.apache.kylin.metadata.model.ISegment;
 import org.apache.kylin.cube.cuboid.Cuboid;
 import org.apache.kylin.cube.kv.RowConstants;
-import org.apache.kylin.dimension.DimensionEncoding;
 import org.apache.kylin.gridtable.GTInfo;
 import org.apache.kylin.gridtable.GTRecord;
 import org.apache.kylin.gridtable.GTScanRequest;
 import org.apache.kylin.gridtable.IGTScanner;
 import org.apache.kylin.gridtable.IGTStore;
-import org.apache.kylin.metadata.filter.UDF.MassInTupleFilter;
-import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.metadata.model.ISegment;
 import 

kylin git commit: Ignore ITKylinQueryTest as it will be invoked by sub-classes

[shaofengshi]

Repository: kylin
Updated Branches:
  refs/heads/master 03118bd44 -> c84892f52


Ignore ITKylinQueryTest as it will be invoked by sub-classes

Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c84892f5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c84892f5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c84892f5

Branch: refs/heads/master
Commit: c84892f527a6a16904d3cb801748e35248b4d90a
Parents: 03118bd
Author: shaofengshi 
Authored: Tue Oct 18 15:14:49 2016 +0800
Committer: shaofengshi 
Committed: Tue Oct 18 15:17:01 2016 +0800

--
 kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java | 1 +
 1 file changed, 1 insertion(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/c84892f5/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java 
b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
index ffd85e3..2ec5324 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
@@ -45,6 +45,7 @@ import org.junit.rules.ExpectedException;
 
 import com.google.common.collect.Maps;
 
+@Ignore("KylinQueryTest is contained by ITCombinationTest")
 public class ITKylinQueryTest extends KylinTestBase {
 
 @Rule

svn commit: r1765396 - in /kylin/site: ./ blog/ blog/2016/10/ blog/2016/10/18/ blog/2016/10/18/new-nrt-streaming/ images/blog/

[lidong]

Author: lidong
Date: Tue Oct 18 07:03:11 2016
New Revision: 1765396

URL: http://svn.apache.org/viewvc?rev=1765396=rev
Log:
Add blog for new streaming

Added:
kylin/site/blog/2016/10/
kylin/site/blog/2016/10/18/
kylin/site/blog/2016/10/18/new-nrt-streaming/
kylin/site/blog/2016/10/18/new-nrt-streaming/index.html
kylin/site/images/blog/new-streaming.png   (with props)
kylin/site/images/blog/offset-as-partition-value.png   (with props)
kylin/site/images/blog/streaming-adapter.png   (with props)
kylin/site/images/blog/streaming-monitor.png   (with props)
kylin/site/images/blog/streaming-twitter.png   (with props)
Modified:
kylin/site/blog/index.html
kylin/site/feed.xml

Added: kylin/site/blog/2016/10/18/new-nrt-streaming/index.html
URL: 
http://svn.apache.org/viewvc/kylin/site/blog/2016/10/18/new-nrt-streaming/index.html?rev=1765396=auto
==
--- kylin/site/blog/2016/10/18/new-nrt-streaming/index.html (added)
+++ kylin/site/blog/2016/10/18/new-nrt-streaming/index.html Tue Oct 18 07:03:11 
2016
@@ -0,0 +1,309 @@
+
+
+
+   
+
+
+  
+  
+  
+
+  Apache Kylin | New NRT Streaming in Apache Kylin
+  
+  
+  
+
+
+
+
+
+
+
+
+
+
+
+
+
+  
+  
+  
+  
+
+  http://kylin.apache.org/blog/2016/10/18/new-nrt-streaming/;>
+  http://kylin.apache.org/feed.xml; />
+
+
+
+  (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
+  (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new 
Date();a=s.createElement(o),
+  
m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
+  })(window,document,'script','//www.google-analytics.com/analytics.js','ga');
+
+  //oringal tracker for kylin.io
+  ga('create', 'UA-55534813-1', 'auto');
+  //new tracker for kylin.apache.org
+  ga('create', 'UA-55534813-2', 'auto', {'name':'toplevel'});
+
+  ga('send', 'pageview');
+  ga('toplevel.send', 'pageview');
+
+
+
+
+ 
+ 
+
+
+   
+   
+
+
+  
+  
+  Apache Kylin™ Extreme OLAP Engine for Big 
Data 
+
+  
+  
+
+  
+  
+  
+
+
+
+  
+Toggle navigation
+
+
+
+  
+ 
+
+
+
+
+  
+ Home
+  Docs
+  Download
+  Community
+  Development
+  Blog
+  中文版  
+  https://twitter.com/apachekylin; target="_blank" 
class="fa fa-twitter fa-lg" title="Twitter: @ApacheKylin" >
+  https://github.com/apache/kylin; target="_blank" 
class="fa fa-github-alt fa-lg" title="Github: apache/kylin" >  
+  https://www.facebook.com/kylinio; target="_blank" 
class="fa fa-facebook fa-lg" title="Facebook: kylin.io" >   
+
+
+  
+
+ 
+
+   
+   
+   
+   Apache Kylin™ 
Technical Blog
+   
+   
+
+   
+   
+
+   
+
+
+
+  
+New NRT Streaming in Apache Kylin
+Oct 18, 2016 • Shaofeng Shi
+  
+
+  
+In 1.5.0 Apache Kylin introduces the Streaming Cubing feature, which 
can consume data from Kafka topic directly. This blog introduces how that be 
implemented, and this tutorial introduces how to use 
it.
+
+While, that implementation was marked as “experimental” because it has 
the following limitations:
+
+
+  
+Not scalable: it starts a Java process for a micro-batch cube building, 
instead of leveraging any computing framework; If too many messages arrive at 
one time, the build may fail with OutOfMemory error;
+  
+  
+May loss data: it uses a time window to seek the approximate start/end 
offsets on Kafka topic, which means too late/early arrived messages will be 
skipped; Then the query couldn’t ensure 100% accuracy.
+  
+  
+Difficult to monitor: the streaming cubing is out of the Job engine’s 
scope, user can not monitor the jobs with Web GUI or REST API.
+  
+  
+Others: hard to recover from accident, difficult to maintain the code, 
etc.
+  
+
+
+To overcome these limitations, the Apache Kylin team developed the new 
streaming (https://issues.apache.org/jira/browse/KYLIN-1726;>KYLIN-1726) with 
Kafka 0.10 API, it has been tested internally for some time, will release to 
public soon.
+
+The new design is a perfect implementation under Kylin 1.5’s 
“Plug-in” architecture: treat Kafka topic as a “Data Source” like Hive 
table, using an adapter to extract the data to HDFS; the next steps are almost 
the same as from Hive. Figure 1 is a high level architecture of the new 
design.
+
+
+
+The adapter to read Kafka messages is modified from https://github.com/amient/kafka-hadoop-loader;>kafka-hadoop-loader, 
which is open sourced under Apache License V2.0; it starts a mapper for each 
Kafka partition, reading and then saving the messages to HDFS; in next steps 
Kylin will be 

kylin git commit: Add blog for new streaming

[shaofengshi]

Repository: kylin
Updated Branches:
  refs/heads/document 8989d052c -> 38cd798d0


Add blog for new streaming

Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/38cd798d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/38cd798d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/38cd798d

Branch: refs/heads/document
Commit: 38cd798d0e656cb202f2d530e6fdd44491f6353a
Parents: 8989d05
Author: shaofengshi 
Authored: Tue Oct 18 14:59:48 2016 +0800
Committer: shaofengshi 
Committed: Tue Oct 18 14:59:48 2016 +0800

--
 .../_posts/blog/2016-10-18-new-nrt-streaming.md |  56 +++
 website/images/blog/new-streaming.png   | Bin 0 -> 152820 bytes
 .../images/blog/offset-as-partition-value.png   | Bin 0 -> 121697 bytes
 website/images/blog/streaming-adapter.png   | Bin 0 -> 117956 bytes
 website/images/blog/streaming-monitor.png   | Bin 0 -> 227754 bytes
 website/images/blog/streaming-twitter.png   | Bin 0 -> 90771 bytes
 6 files changed, 56 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/38cd798d/website/_posts/blog/2016-10-18-new-nrt-streaming.md
--
diff --git a/website/_posts/blog/2016-10-18-new-nrt-streaming.md 
b/website/_posts/blog/2016-10-18-new-nrt-streaming.md
new file mode 100644
index 000..e1cb845
--- /dev/null
+++ b/website/_posts/blog/2016-10-18-new-nrt-streaming.md
@@ -0,0 +1,56 @@
+---
+layout: post-blog
+title:  New NRT Streaming in Apache Kylin
+date:   2016-10-18 17:30:00
+author: Shaofeng Shi
+categories: blog
+---
+
+In 1.5.0 Apache Kylin introduces the Streaming Cubing feature, which can 
consume data from Kafka topic directly. This 
[blog](/blog/2016/02/03/streaming-cubing/) introduces how that be implemented, 
and this [tutorial](/docs15/tutorial/cube_streaming.html) introduces how to use 
it.
+
+While, that implementation was marked as "experimental" because it has the 
following limitations:
+
+ * Not scalable: it starts a Java process for a micro-batch cube building, 
instead of leveraging any computing framework; If too many messages arrive at 
one time, the build may fail with OutOfMemory error;
+
+ * May loss data: it uses a time window to seek the approximate start/end 
offsets on Kafka topic, which means too late/early arrived messages will be 
skipped; Then the query couldn't ensure 100% accuracy.
+
+ * Difficult to monitor: the streaming cubing is out of the Job engine's 
scope, user can not monitor the jobs with Web GUI or REST API. 
+
+ * Others: hard to recover from accident, difficult to maintain the code, etc.
+
+To overcome these limitations, the Apache Kylin team developed the new 
streaming ([KYLIN-1726](https://issues.apache.org/jira/browse/KYLIN-1726)) with 
Kafka 0.10 API, it has been tested internally for some time, will release to 
public soon.
+
+The new design is a perfect implementation under Kylin 1.5's "Plug-in" 
architecture: treat Kafka topic as a "Data Source" like Hive table, using an 
adapter to extract the data to HDFS; the next steps are almost the same as from 
Hive. Figure 1 is a high level architecture of the new design.
+
+
+![Kylin New Streaming Framework Architecture](/images/blog/new-streaming.png)
+
+The adapter to read Kafka messages is modified from 
[kafka-hadoop-loader](https://github.com/amient/kafka-hadoop-loader), which is 
open sourced under Apache License V2.0; it starts a mapper for each Kafka 
partition, reading and then saving the messages to HDFS; in next steps Kylin 
will be able to leverage existing framework like MR to do the processing, this 
makes the solution scalable and fault-tolerant. 
+
+To overcome the "data loss" problem, Kylin adds the start/end offset 
information on each Cube segment, and then use the offsets as the partition 
value (no overlap is allowed); this ensures no data be lost and 1 message be 
consumed at most once. To let the late/early message can be queried, Cube 
segments allow overlap for the partition time dimension: Kylin will scan all 
segments which include the queried time. Figure 2 illurates this.
+
+![Use Offset to Cut Segments](/images/blog/offset-as-partition-value.png)
+
+Other changes/enhancements are made in the new streaming:
+
+ * Allow multiple segments being built/merged concurrently
+ * Automatically seek start/end offsets (if user doesn't specify) from 
previous segment or Kafka
+ * Support embeded properties in JSON message
+ * Add REST API to trigger streaming cube's building
+ * Add REST API to check and fill the segment holes
+
+The integration test result shows big improvements than the previous version:
+
+ * Scalability: it can easily process up to hundreds of million records in one 
build; 
+ *