incubator-kylin git commit: bug fix

2015-11-05 Thread mahongbin 
Repository: incubator-kylin
Updated Branches:
  refs/heads/KYLIN-1126 b6f6311bc -> 74536a8e6


bug fix


Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/74536a8e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/74536a8e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/74536a8e

Branch: refs/heads/KYLIN-1126
Commit: 74536a8e6addde7155f050f2b4c7d60e5f7b29f4
Parents: b6f6311
Author: honma 
Authored: Thu Nov 5 17:37:40 2015 +0800
Committer: honma 
Committed: Thu Nov 5 17:37:40 2015 +0800

--
 .../org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java| 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/74536a8e/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java
index b63b1f3..31cce7b 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java
@@ -94,7 +94,7 @@ public final class HBaseCuboidWriter implements ICuboidWriter 
{
 
 //TODO:shardingonstreaming
 private byte[] createKey(Long cuboidId, GTRecord record) {
-if (rowKeyEncoder.getCuboidID() != cuboidId) {
+if (rowKeyEncoder == null || rowKeyEncoder.getCuboidID() != cuboidId) {
 rowKeyEncoder = AbstractRowKeyEncoder.createInstance(cubeSegment, 
Cuboid.findById(cubeDesc, cuboidId));
 keybuf = rowKeyEncoder.createBuf();
 }

[04/31] incubator-kylin git commit: bug fix, SingleValueCache.put() should put local cache first

2015-11-04 Thread shaofengshi 
bug fix, SingleValueCache.put() should put local cache first


Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/b9e64da6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/b9e64da6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/b9e64da6

Branch: refs/heads/KYLIN-1112
Commit: b9e64da6434a2c163bcfd5066d411299763e1e6a
Parents: 730739f
Author: Li, Yang 
Authored: Thu Oct 29 11:50:31 2015 +0800
Committer: shaofengshi 
Committed: Thu Nov 5 09:53:15 2015 +0800

--
 .../java/org/apache/kylin/common/restclient/SingleValueCache.java  | 2 ++
 .../src/main/java/org/apache/kylin/metadata/MetadataManager.java   | 2 +-
 2 files changed, 3 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b9e64da6/core-common/src/main/java/org/apache/kylin/common/restclient/SingleValueCache.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/restclient/SingleValueCache.java
 
b/core-common/src/main/java/org/apache/kylin/common/restclient/SingleValueCache.java
index 50b0f84..2a3ff06 100644
--- 
a/core-common/src/main/java/org/apache/kylin/common/restclient/SingleValueCache.java
+++ 
b/core-common/src/main/java/org/apache/kylin/common/restclient/SingleValueCache.java
@@ -44,6 +44,8 @@ public abstract class SingleValueCache extends 
AbstractRestCache {
 
 public void put(K key, V value) {
 boolean exists = innerCache.containsKey(key);
+
+innerCache.put(key, value);
 
 if (!exists) {
 cacheUpdater.updateCache(key, value, Broadcaster.EVENT.CREATE, 
syncType, this);

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b9e64da6/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java 
b/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
index 9b66578..b7e7dc5 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
@@ -431,7 +431,7 @@ public class MetadataManager {
 os.close();
 is.close();
 
-srcTableExdMap.putLocal(tableId, tableExdProperties);
+srcTableExdMap.put(tableId, tableExdProperties);
 }
 
 public String appendDBName(String table) {

incubator-kylin git commit: bug fix

2015-10-28 Thread mahongbin 
Repository: incubator-kylin
Updated Branches:
  refs/heads/KYLIN-942-x 342ee79e0 -> 7e1f084e2


bug fix


Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/7e1f084e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/7e1f084e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/7e1f084e

Branch: refs/heads/KYLIN-942-x
Commit: 7e1f084e2f6d2c0fa988fe7032f3b4dc7b3c9f85
Parents: 342ee79
Author: honma 
Authored: Wed Oct 28 15:11:54 2015 +0800
Committer: honma 
Committed: Wed Oct 28 15:11:54 2015 +0800

--
 .../java/org/apache/kylin/gridtable/GTScanRange.java | 15 +--
 1 file changed, 5 insertions(+), 10 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7e1f084e/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRange.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRange.java 
b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRange.java
index d5fa6c0..87f51df 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRange.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRange.java
@@ -3,9 +3,6 @@ package org.apache.kylin.gridtable;
 import java.util.Collections;
 import java.util.List;
 
-import javax.annotation.Nullable;
-
-import com.google.common.base.Function;
 import com.google.common.collect.Lists;
 
 public class GTScanRange {
@@ -28,15 +25,13 @@ public class GTScanRange {
 }
 
 public GTScanRange replaceGTInfo(final GTInfo gtInfo) {
+List newFuzzyKeys = Lists.newArrayList();
+for (GTRecord input : fuzzyKeys) {
+newFuzzyKeys.add(new GTRecord(gtInfo, 
input.maskForEqualHashComp(), input.cols));
+}
 return new GTScanRange(new GTRecord(gtInfo, 
pkStart.maskForEqualHashComp(), pkStart.cols), //
 new GTRecord(gtInfo, pkEnd.maskForEqualHashComp(), 
pkEnd.cols), //
-Lists.transform(fuzzyKeys, new Function() {
-@Nullable
-@Override
-public GTRecord apply(GTRecord input) {
-return new GTRecord(gtInfo, 
input.maskForEqualHashComp(), input.cols);
-}
-}));
+newFuzzyKeys);
 }
 
 @Override

incubator-kylin git commit: bug fix, SingleValueCache.put() should put local cache first

2015-10-28 Thread liyang 
Repository: incubator-kylin
Updated Branches:
  refs/heads/2.x-staging b64a0e998 -> cb94c8127


bug fix, SingleValueCache.put() should put local cache first


Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/cb94c812
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/cb94c812
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/cb94c812

Branch: refs/heads/2.x-staging
Commit: cb94c812707bc53b7d5da78f96f68e046400a29b
Parents: b64a0e9
Author: Li, Yang 
Authored: Thu Oct 29 11:50:31 2015 +0800
Committer: Li, Yang 
Committed: Thu Oct 29 11:51:18 2015 +0800

--
 .../java/org/apache/kylin/common/restclient/SingleValueCache.java  | 2 ++
 .../src/main/java/org/apache/kylin/metadata/MetadataManager.java   | 2 +-
 2 files changed, 3 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/cb94c812/core-common/src/main/java/org/apache/kylin/common/restclient/SingleValueCache.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/restclient/SingleValueCache.java
 
b/core-common/src/main/java/org/apache/kylin/common/restclient/SingleValueCache.java
index 50b0f84..2a3ff06 100644
--- 
a/core-common/src/main/java/org/apache/kylin/common/restclient/SingleValueCache.java
+++ 
b/core-common/src/main/java/org/apache/kylin/common/restclient/SingleValueCache.java
@@ -44,6 +44,8 @@ public abstract class SingleValueCache extends 
AbstractRestCache {
 
 public void put(K key, V value) {
 boolean exists = innerCache.containsKey(key);
+
+innerCache.put(key, value);
 
 if (!exists) {
 cacheUpdater.updateCache(key, value, Broadcaster.EVENT.CREATE, 
syncType, this);

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/cb94c812/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java 
b/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
index 9b66578..b7e7dc5 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
@@ -431,7 +431,7 @@ public class MetadataManager {
 os.close();
 is.close();
 
-srcTableExdMap.putLocal(tableId, tableExdProperties);
+srcTableExdMap.put(tableId, tableExdProperties);
 }
 
 public String appendDBName(String table) {

incubator-kylin git commit: bug

2015-10-22 Thread mahongbin 
Repository: incubator-kylin
Updated Branches:
  refs/heads/KYLIN-942 cd81a8d1b -> e9067ef86


bug


Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/e9067ef8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/e9067ef8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/e9067ef8

Branch: refs/heads/KYLIN-942
Commit: e9067ef86b2345ed26c2b60d4e7478edd74b8ade
Parents: cd81a8d
Author: honma 
Authored: Thu Oct 22 15:06:01 2015 +0800
Committer: honma 
Committed: Thu Oct 22 15:06:01 2015 +0800

--
 .../apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java   | 4 ++--
 .../hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/e9067ef8/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java
index 4721cb6..286da55 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java
@@ -222,8 +222,8 @@ public class CubeSegmentScanner implements IGTScanner {
 
 try {
 
-//CubeHBaseRPC rpc = new 
CubeHBaseEndpointRPC(cubeSeg, cuboid, info);
-CubeHBaseRPC rpc = new CubeHBaseScanRPC(cubeSeg, 
cuboid, info);
+CubeHBaseRPC rpc = new 
CubeHBaseEndpointRPC(cubeSeg, cuboid, info);
+//CubeHBaseRPC rpc = new CubeHBaseScanRPC(cubeSeg, 
cuboid, info);
 
 //change previous line to CubeHBaseRPC rpc = new 
CubeHBaseScanRPC(cubeSeg, cuboid, info);
 //to debug locally

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/e9067ef8/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
index 356d6e0..2b7c20c 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
@@ -142,7 +142,7 @@ public class CubeVisitService extends 
CubeVisitProtos.CubeVisitService implement
 innerScanner = region.getScanner(scan);
 InnerScannerAsIterator cellListIterator = new 
InnerScannerAsIterator(innerScanner);
 
-IGTStore store = new HBaseReadonlyStore(cellListIterator, scanReq, 
hbaseRawScan.hbaseColumns, null);
+IGTStore store = new HBaseReadonlyStore(cellListIterator, scanReq, 
hbaseRawScan.hbaseColumns, hbaseColumnsToGT);
 IGTScanner rawScanner = store.scan(scanReq);
 IGTScanner finalScanner = scanReq.decorateScanner(rawScanner);

incubator-kylin git commit: bug fix for ci

2015-09-07 Thread mahongbin 
Repository: incubator-kylin
Updated Branches:
  refs/heads/0.8 7797c2bfb -> 9f4b93b94


bug fix for ci


Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/9f4b93b9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/9f4b93b9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/9f4b93b9

Branch: refs/heads/0.8
Commit: 9f4b93b949d93bcb1ca0e4afd22e36d4016ccae3
Parents: 7797c2b
Author: honma 
Authored: Mon Sep 7 14:38:56 2015 +0800
Committer: honma 
Committed: Mon Sep 7 14:38:56 2015 +0800

--
 .../java/org/apache/kylin/metadata/model/DataModelDesc.java | 9 +
 1 file changed, 5 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9f4b93b9/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
index 1c6ef62..66581c0 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
@@ -104,7 +104,7 @@ public class DataModelDesc extends RootPersistentEntity {
 public String getFactTable() {
 return factTable;
 }
-
+
 public TableDesc getFactTableDesc() {
 return factTableDesc;
 }
@@ -175,6 +175,7 @@ public class DataModelDesc extends RootPersistentEntity {
 }
 
 public void init(Map tables) {
+this.factTable = this.factTable.toUpperCase();
 this.factTableDesc = tables.get(this.factTable.toUpperCase());
 if (factTableDesc == null) {
 throw new IllegalStateException("Fact table does not exist:" + 
this.factTable);
@@ -207,7 +208,7 @@ public class DataModelDesc extends RootPersistentEntity {
 
 StringUtil.toUpperCaseArray(join.getForeignKey(), 
join.getForeignKey());
 StringUtil.toUpperCaseArray(join.getPrimaryKey(), 
join.getPrimaryKey());
-
+
 // primary key
 String[] pks = join.getPrimaryKey();
 TblColRef[] pkCols = new TblColRef[pks.length];
@@ -221,7 +222,7 @@ public class DataModelDesc extends RootPersistentEntity {
 pkCols[i] = colRef;
 }
 join.setPrimaryKeyColumns(pkCols);
-
+
 // foreign key
 String[] fks = join.getForeignKey();
 TblColRef[] fkCols = new TblColRef[fks.length];
@@ -235,7 +236,7 @@ public class DataModelDesc extends RootPersistentEntity {
 fkCols[i] = colRef;
 }
 join.setForeignKeyColumns(fkCols);
-
+
 // Validate join in dimension
 if (pkCols.length != fkCols.length) {
 throw new IllegalStateException("Primary keys(" + 
lookup.getTable() + ")" + Arrays.toString(pks) + " are not consistent with 
Foreign keys(" + this.getFactTable() + ") " + Arrays.toString(fks));

[25/28] incubator-kylin git commit: bug fix for ci

2015-09-07 Thread mahongbin 
bug fix for ci


Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/9f4b93b9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/9f4b93b9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/9f4b93b9

Branch: refs/heads/2.x-staging
Commit: 9f4b93b949d93bcb1ca0e4afd22e36d4016ccae3
Parents: 7797c2b
Author: honma 
Authored: Mon Sep 7 14:38:56 2015 +0800
Committer: honma 
Committed: Mon Sep 7 14:38:56 2015 +0800

--
 .../java/org/apache/kylin/metadata/model/DataModelDesc.java | 9 +
 1 file changed, 5 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9f4b93b9/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
index 1c6ef62..66581c0 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
@@ -104,7 +104,7 @@ public class DataModelDesc extends RootPersistentEntity {
 public String getFactTable() {
 return factTable;
 }
-
+
 public TableDesc getFactTableDesc() {
 return factTableDesc;
 }
@@ -175,6 +175,7 @@ public class DataModelDesc extends RootPersistentEntity {
 }
 
 public void init(Map tables) {
+this.factTable = this.factTable.toUpperCase();
 this.factTableDesc = tables.get(this.factTable.toUpperCase());
 if (factTableDesc == null) {
 throw new IllegalStateException("Fact table does not exist:" + 
this.factTable);
@@ -207,7 +208,7 @@ public class DataModelDesc extends RootPersistentEntity {
 
 StringUtil.toUpperCaseArray(join.getForeignKey(), 
join.getForeignKey());
 StringUtil.toUpperCaseArray(join.getPrimaryKey(), 
join.getPrimaryKey());
-
+
 // primary key
 String[] pks = join.getPrimaryKey();
 TblColRef[] pkCols = new TblColRef[pks.length];
@@ -221,7 +222,7 @@ public class DataModelDesc extends RootPersistentEntity {
 pkCols[i] = colRef;
 }
 join.setPrimaryKeyColumns(pkCols);
-
+
 // foreign key
 String[] fks = join.getForeignKey();
 TblColRef[] fkCols = new TblColRef[fks.length];
@@ -235,7 +236,7 @@ public class DataModelDesc extends RootPersistentEntity {
 fkCols[i] = colRef;
 }
 join.setForeignKeyColumns(fkCols);
-
+
 // Validate join in dimension
 if (pkCols.length != fkCols.length) {
 throw new IllegalStateException("Primary keys(" + 
lookup.getTable() + ")" + Arrays.toString(pks) + " are not consistent with 
Foreign keys(" + this.getFactTable() + ") " + Arrays.toString(fks));

[1/3] incubator-kylin git commit: bug fix: cube_statistics file ts not right

2015-08-11 Thread mahongbin 
Repository: incubator-kylin
Updated Branches:
  refs/heads/0.8 05947f31e - ce4511b7a


bug fix: cube_statistics file ts not right


Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/ce4511b7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/ce4511b7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/ce4511b7

Branch: refs/heads/0.8
Commit: ce4511b7a78b95801fb729180a13691f553d8837
Parents: 5c33d5a
Author: honma ho...@ebay.com
Authored: Wed Aug 12 10:18:02 2015 +0800
Committer: honma ho...@ebay.com
Committed: Wed Aug 12 10:18:18 2015 +0800

--
 .../kylin/job/streaming/CubeStreamConsumer.java | 28 +
 .../storage/hbase/HBaseStreamingOutput.java | 60 +++-
 2 files changed, 40 insertions(+), 48 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ce4511b7/job/src/main/java/org/apache/kylin/job/streaming/CubeStreamConsumer.java
--
diff --git 
a/job/src/main/java/org/apache/kylin/job/streaming/CubeStreamConsumer.java 
b/job/src/main/java/org/apache/kylin/job/streaming/CubeStreamConsumer.java
index 5670224..4a909ac 100644
--- a/job/src/main/java/org/apache/kylin/job/streaming/CubeStreamConsumer.java
+++ b/job/src/main/java/org/apache/kylin/job/streaming/CubeStreamConsumer.java
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
 package org.apache.kylin.job.streaming;
 
 import java.io.IOException;
@@ -9,7 +26,6 @@ import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.LinkedBlockingQueue;
 
-import com.google.common.collect.Maps;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FileSystem;
@@ -37,20 +53,12 @@ import org.apache.kylin.source.ReadableTable.TableSignature;
 import org.apache.kylin.storage.hbase.HBaseConnection;
 import org.apache.kylin.storage.hbase.HBaseCuboidWriter;
 import org.apache.kylin.storage.hbase.steps.CubeHTableUtil;
-import org.apache.kylin.storage.hbase.steps.InMemKeyValueCreator;
 import org.apache.kylin.streaming.MicroStreamBatch;
 import org.apache.kylin.streaming.MicroStreamBatchConsumer;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.*;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.LinkedBlockingQueue;
 
 /**
  */
@@ -96,7 +104,7 @@ public class CubeStreamConsumer implements 
MicroStreamBatchConsumer {
 FileSystem.getLocal(conf).deleteOnExit(outputPath);
 FactDistinctColumnsReducer.writeCuboidStatistics(conf, outputPath, 
samplingResult, 100);
 FSDataInputStream localStream = FileSystem.getLocal(conf).open(new 
Path(outputPath, BatchConstants.CFG_STATISTICS_CUBOID_ESTIMATION));
-
ResourceStore.getStore(kylinConfig).putResource(cubeSegment.getStatisticsResourcePath(),
 localStream, 0);
+
ResourceStore.getStore(kylinConfig).putResource(cubeSegment.getStatisticsResourcePath(),
 localStream, System.currentTimeMillis());
 localStream.close();
 
 final MapTblColRef, Dictionary? dictionaryMap = 
CubingUtils.buildDictionary(cubeInstance, parsedStreamMessages);

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ce4511b7/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseStreamingOutput.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseStreamingOutput.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseStreamingOutput.java
index 10291fa..7d10b3d 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseStreamingOutput.java
+++

incubator-kylin git commit: bug fix: Memory budget

2015-06-16 Thread mahongbin 
Repository: incubator-kylin
Updated Branches:
  refs/heads/0.8 49e722ec2 - f17a072f3


bug fix: Memory budget


Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/f17a072f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/f17a072f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/f17a072f

Branch: refs/heads/0.8
Commit: f17a072f385921d3277ec10cda670b4e59f97e41
Parents: 49e722e
Author: honma ho...@ebay.com
Authored: Tue Jun 16 20:23:15 2015 +0800
Committer: honma ho...@ebay.com
Committed: Tue Jun 16 20:23:15 2015 +0800

--
 .../kylin/job/inmemcubing/InMemCubeBuilder.java | 47 ++--
 1 file changed, 13 insertions(+), 34 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f17a072f/job/src/main/java/org/apache/kylin/job/inmemcubing/InMemCubeBuilder.java
--
diff --git 
a/job/src/main/java/org/apache/kylin/job/inmemcubing/InMemCubeBuilder.java 
b/job/src/main/java/org/apache/kylin/job/inmemcubing/InMemCubeBuilder.java
index 5833b0f..b9c48aa 100644
--- a/job/src/main/java/org/apache/kylin/job/inmemcubing/InMemCubeBuilder.java
+++ b/job/src/main/java/org/apache/kylin/job/inmemcubing/InMemCubeBuilder.java
@@ -16,32 +16,12 @@
  */
 package org.apache.kylin.job.inmemcubing;
 
-import java.io.Closeable;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.BitSet;
-import java.util.Collections;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.SortedMap;
-import java.util.TreeMap;
-import java.util.TreeSet;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.LinkedBlockingDeque;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
-
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
 import org.apache.hadoop.io.DoubleWritable;
 import org.apache.hadoop.io.LongWritable;
 import org.apache.kylin.common.hll.HyperLogLogPlusCounter;
-import org.apache.kylin.common.util.ByteArray;
-import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.common.util.ImmutableBitSet;
-import org.apache.kylin.common.util.Pair;
+import org.apache.kylin.common.util.*;
 import org.apache.kylin.cube.cuboid.Cuboid;
 import org.apache.kylin.cube.cuboid.CuboidScheduler;
 import org.apache.kylin.cube.model.CubeDesc;
@@ -54,19 +34,18 @@ import org.apache.kylin.metadata.model.FunctionDesc;
 import org.apache.kylin.metadata.model.MeasureDesc;
 import org.apache.kylin.metadata.model.TblColRef;
 import org.apache.kylin.storage.cube.CubeGridTable;
-import org.apache.kylin.storage.gridtable.GTAggregateScanner;
-import org.apache.kylin.storage.gridtable.GTBuilder;
-import org.apache.kylin.storage.gridtable.GTInfo;
-import org.apache.kylin.storage.gridtable.GTRecord;
-import org.apache.kylin.storage.gridtable.GTScanRequest;
-import org.apache.kylin.storage.gridtable.GridTable;
-import org.apache.kylin.storage.gridtable.IGTScanner;
-import org.apache.kylin.storage.gridtable.IGTStore;
+import org.apache.kylin.storage.gridtable.*;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
+import java.io.Closeable;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.*;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.LinkedBlockingDeque;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
 
 /**
  */
@@ -360,7 +339,7 @@ public class InMemCubeBuilder implements Runnable {
 private void makeMemoryBudget() {
 int systemAvailMB = getSystemAvailMB();
 logger.info(System avail  + systemAvailMB +  MB);
-int reserve = Math.min(100, baseResult.aggrCacheMB / 3);
+int reserve = Math.max(100, baseResult.aggrCacheMB / 3);
 logger.info(Reserve  + reserve +  MB for system basics);
 
 int budget = systemAvailMB - reserve;

incubator-kylin git commit: bug fix on long running cube steaming

2015-06-09 Thread mahongbin 
Repository: incubator-kylin
Updated Branches:
  refs/heads/0.8.0 02b0d84a0 - 247d4f13c


bug fix on long running cube steaming


Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/247d4f13
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/247d4f13
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/247d4f13

Branch: refs/heads/0.8.0
Commit: 247d4f13c5bc0d0f910e041bdd34f11e1483fe08
Parents: 02b0d84
Author: honma ho...@ebay.com
Authored: Tue Jun 9 19:07:51 2015 +0800
Committer: honma ho...@ebay.com
Committed: Tue Jun 9 19:08:03 2015 +0800

--
 .../org/apache/kylin/common/util/BasicTest.java |  2 +-
 .../java/org/apache/kylin/cube/CubeManager.java | 20 
 .../apache/kylin/query/routing/QueryRouter.java | 12 ++--
 .../apache/kylin/rest/service/CacheService.java |  1 +
 .../apache/kylin/rest/service/CubeService.java  |  2 ++
 .../kylin/storage/hbase/CubeStorageEngine.java  |  9 +++--
 6 files changed, 33 insertions(+), 13 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/247d4f13/common/src/test/java/org/apache/kylin/common/util/BasicTest.java
--
diff --git a/common/src/test/java/org/apache/kylin/common/util/BasicTest.java 
b/common/src/test/java/org/apache/kylin/common/util/BasicTest.java
index 73f708d..01a22c1 100644
--- a/common/src/test/java/org/apache/kylin/common/util/BasicTest.java
+++ b/common/src/test/java/org/apache/kylin/common/util/BasicTest.java
@@ -77,7 +77,7 @@ public class BasicTest {
 @Test
 @Ignore(convenient trial tool for dev)
 public void test1() throws Exception {
-
System.out.println(org.apache.kylin.common.util.DateFormat.formatToTimeStr(1433496233000L));
+
System.out.println(org.apache.kylin.common.util.DateFormat.formatToTimeStr(1433833611000L));
 
System.out.println(org.apache.kylin.common.util.DateFormat.formatToTimeStr(1433250517000L));
 
System.out.println(org.apache.kylin.common.util.DateFormat.stringToMillis(2015-06-01
 00:00:00));
 
System.out.println(org.apache.kylin.common.util.DateFormat.stringToMillis(2015-05-15
 17:00:00));

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/247d4f13/cube/src/main/java/org/apache/kylin/cube/CubeManager.java
--
diff --git a/cube/src/main/java/org/apache/kylin/cube/CubeManager.java 
b/cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index ac49dfa..29bc9b0 100644
--- a/cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -18,6 +18,8 @@
 
 package org.apache.kylin.cube;
 
+import com.google.common.base.Function;
+import com.google.common.collect.Collections2;
 import com.google.common.collect.HashMultimap;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Multimap;
@@ -48,6 +50,7 @@ import org.apache.kylin.metadata.realization.*;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import javax.annotation.Nullable;
 import java.io.IOException;
 import java.util.*;
 import java.util.concurrent.ConcurrentHashMap;
@@ -351,7 +354,7 @@ public class CubeManager implements IRealizationProvider {
 return appendSegments(cube, startDate, endDate, strictChecking, 
saveChange);
 }
 
-public CubeSegment appendSegments(CubeInstance cube, long startDate,  long 
endDate, boolean strictChecking, boolean saveChange) throws IOException {
+public CubeSegment appendSegments(CubeInstance cube, long startDate, long 
endDate, boolean strictChecking, boolean saveChange) throws IOException {
 if (strictChecking)
 checkNoBuildingSegment(cube);
 
@@ -385,7 +388,7 @@ public class CubeManager implements IRealizationProvider {
 PairLong, Long range = alignMergeRange(cube, startDate, endDate);
 CubeSegment newSegment = newSegment(cube, range.getFirst(), 
range.getSecond());
 
-validateNewSegments(cube, newSegment);
+validateNewSegments(cube, false, newSegment);
 
 CubeBuilder cubeBuilder = new CubeBuilder(cube);
 cubeBuilder.setToAddSegs(newSegment);
@@ -553,7 +556,7 @@ public class CubeManager implements IRealizationProvider {
 for (CubeSegment segment : readySegments) {
 long thisSegmentRange = segment.getDateRangeEnd() - 
segment.getDateRangeStart();
 
-if (thisSegmentRange = toMergeRange ) {
+if (thisSegmentRange = toMergeRange) {
 // this segment and its previous segments will not be 
merged
 toMergeSegments.clear();
 currentRange = 0;
@@ -658,7 +661,7

[2/2] incubator-kylin git commit: bug fix

2015-06-02 Thread mahongbin 
bug fix


Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/487c0af0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/487c0af0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/487c0af0

Branch: refs/heads/0.8.0
Commit: 487c0af027ade5603fe7a951cb4ff2f80c5aa12e
Parents: 6d6a7c1
Author: honma ho...@ebay.com
Authored: Tue Jun 2 18:23:11 2015 +0800
Committer: honma ho...@ebay.com
Committed: Tue Jun 2 22:25:55 2015 +0800

--
 .../org/apache/kylin/job/streaming/CubeStreamBuilder.java   | 4 ++--
 .../org/apache/kylin/job/streaming/StreamingBootstrap.java  | 6 +-
 .../main/java/org/apache/kylin/streaming/StreamBuilder.java | 9 +++--
 3 files changed, 14 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/487c0af0/job/src/main/java/org/apache/kylin/job/streaming/CubeStreamBuilder.java
--
diff --git 
a/job/src/main/java/org/apache/kylin/job/streaming/CubeStreamBuilder.java 
b/job/src/main/java/org/apache/kylin/job/streaming/CubeStreamBuilder.java
index 2831caa..3c98464 100644
--- a/job/src/main/java/org/apache/kylin/job/streaming/CubeStreamBuilder.java
+++ b/job/src/main/java/org/apache/kylin/job/streaming/CubeStreamBuilder.java
@@ -370,11 +370,11 @@ public class CubeStreamBuilder extends StreamBuilder {
 
 @Override
 protected int batchInterval() {
-return 5 * 60 * 1000;//30 min
+return 5 * 60 * 1000;//5 min
 }
 
 @Override
 protected int batchSize() {
-return 1000;
+return 1;
 }
 }

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/487c0af0/job/src/main/java/org/apache/kylin/job/streaming/StreamingBootstrap.java
--
diff --git 
a/job/src/main/java/org/apache/kylin/job/streaming/StreamingBootstrap.java 
b/job/src/main/java/org/apache/kylin/job/streaming/StreamingBootstrap.java
index ee00880..dcfa774 100644
--- a/job/src/main/java/org/apache/kylin/job/streaming/StreamingBootstrap.java
+++ b/job/src/main/java/org/apache/kylin/job/streaming/StreamingBootstrap.java
@@ -130,7 +130,7 @@ public class StreamingBootstrap {
 
 private ListBlockingQueueStreamMessage consume(KafkaConfig 
kafkaConfig, final int partitionCount) {
 ListBlockingQueueStreamMessage result = Lists.newArrayList();
-for (int partitionId = 0; partitionId  partitionCount  partitionId 
 3; ++partitionId) {
+for (int partitionId = 0; partitionId  partitionCount; ++partitionId) 
{
 final Broker leadBroker = getLeadBroker(kafkaConfig, partitionId);
 
 final long latestOffset = 
KafkaRequester.getLastOffset(kafkaConfig.getTopic(), partitionId, 
OffsetRequest.LatestTime(), leadBroker, kafkaConfig);
@@ -153,10 +153,14 @@ public class StreamingBootstrap {
 Executors.newSingleThreadExecutor().execute(new Runnable() {
 @Override
 public void run() {
+int totalMessage = 0;
 while (true) {
 for (BlockingQueueStreamMessage queue : queues) {
 try {
 streamQueue.put(queue.take());
+if (totalMessage++ % 1 == 1) {
+logger.info(Total stream message count:  + 
totalMessage);
+}
 } catch (InterruptedException e) {
 throw new RuntimeException(e);
 }

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/487c0af0/streaming/src/main/java/org/apache/kylin/streaming/StreamBuilder.java
--
diff --git 
a/streaming/src/main/java/org/apache/kylin/streaming/StreamBuilder.java 
b/streaming/src/main/java/org/apache/kylin/streaming/StreamBuilder.java
index cb5dc1d..3008722 100644
--- a/streaming/src/main/java/org/apache/kylin/streaming/StreamBuilder.java
+++ b/streaming/src/main/java/org/apache/kylin/streaming/StreamBuilder.java
@@ -35,7 +35,6 @@
 package org.apache.kylin.streaming;
 
 import com.google.common.collect.Lists;
-import org.apache.commons.collections.CollectionUtils;
 import org.apache.kylin.common.util.Pair;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -50,7 +49,6 @@ public abstract class StreamBuilder implements Runnable {
 
 private static final Logger logger = 
LoggerFactory.getLogger(StreamBuilder.class);
 
-
 private StreamParser streamParser = StringStreamParser.instance;
 
 private StreamFilter streamFilter = DefaultStreamFilter.instance;
@@ -84,6 +82,7 @@ public abstract class StreamBuilder implements Runnable

[35/50] incubator-kylin git commit: bug fix, snapshot serialize to accommodate NULL value

2015-04-03 Thread liyang 
bug fix, snapshot serialize to accommodate NULL value


Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/99448fec
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/99448fec
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/99448fec

Branch: refs/heads/streaming-localdict
Commit: 99448fecdb58546195373e80ad6bd742f2e47f82
Parents: dc441a6
Author: Li, Yang yang...@ebay.com
Authored: Fri Apr 3 16:22:50 2015 +0800
Committer: Li, Yang yang...@ebay.com
Committed: Fri Apr 3 16:22:50 2015 +0800

--
 .../apache/kylin/dict/lookup/SnapshotTable.java   | 18 +-
 1 file changed, 17 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/99448fec/dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java
--
diff --git 
a/dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java 
b/dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java
index faa4fe6..5bd66ba 100644
--- a/dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java
+++ b/dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java
@@ -21,6 +21,7 @@ package org.apache.kylin.dict.lookup;
 import java.io.DataInput;
 import java.io.DataOutput;
 import java.io.IOException;
+import java.io.UnsupportedEncodingException;
 import java.util.ArrayList;
 import java.util.Arrays;
 
@@ -30,6 +31,7 @@ import org.apache.hadoop.fs.Path;
 import com.fasterxml.jackson.annotation.JsonAutoDetect;
 import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
 import com.fasterxml.jackson.annotation.JsonProperty;
+
 import org.apache.kylin.common.persistence.ResourceStore;
 import org.apache.kylin.common.persistence.RootPersistentEntity;
 import org.apache.kylin.metadata.model.TableDesc;
@@ -149,6 +151,16 @@ public class SnapshotTable extends RootPersistentEntity 
implements ReadableTable
 return true;
 }
 
+private static String NULL_STR;
+{
+try {
+// a special placeholder to indicate a NULL; 0, 1, 9, 127 are a 
few invisible ASCII characters
+NULL_STR = new String(new byte[] { 0, 1, 9, 127 }, ISO-8859-1);
+} catch (UnsupportedEncodingException e) {
+// does not happen
+}
+}
+
 void writeData(DataOutput out) throws IOException {
 out.writeInt(rows.size());
 if (rows.size()  0) {
@@ -157,7 +169,8 @@ public class SnapshotTable extends RootPersistentEntity 
implements ReadableTable
 for (int i = 0; i  rows.size(); i++) {
 String[] row = rows.get(i);
 for (int j = 0; j  n; j++) {
-out.writeUTF(row[j]);
+// NULL_STR is tricky, but we don't want to break the 
current snapshots
+out.writeUTF(row[j] == null ? NULL_STR : row[j]);
 }
 }
 }
@@ -173,6 +186,9 @@ public class SnapshotTable extends RootPersistentEntity 
implements ReadableTable
 rows.add(row);
 for (int j = 0; j  n; j++) {
 row[j] = in.readUTF();
+// NULL_STR is tricky, but we don't want to break the 
current snapshots
+if (row[j].equals(NULL_STR))
+row[j] = null;
 }
 }
 }