[3/3] kylin git commit: refactor

[liyang]

refactor


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a7a318c2
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a7a318c2
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a7a318c2

Branch: refs/heads/mhb0220
Commit: a7a318c214a8b03d8a133b7ec035765fceb5f734
Parents: 661f016
Author: Hongbin Ma 
Authored: Mon Feb 20 10:15:53 2017 +0800
Committer: Hongbin Ma 
Committed: Tue Feb 21 20:55:36 2017 +0800

--
 .../cube/inmemcubing/ConcurrentDiskStore.java   |  7 +
 .../cube/inmemcubing/InMemCubeBuilder.java  |  5 ---
 .../kylin/cube/inmemcubing/MemDiskStore.java|  6 
 .../apache/kylin/gridtable/EmptyGTScanner.java  |  9 +-
 .../kylin/gridtable/GTAggregateScanner.java |  7 +
 .../apache/kylin/gridtable/GTFilterScanner.java |  5 ---
 .../apache/kylin/gridtable/GTScanRequest.java   |  9 +++---
 .../org/apache/kylin/gridtable/IGTScanner.java  |  4 ---
 .../benchmark/SortedGTRecordGenerator.java  |  6 
 .../gridtable/memstore/GTSimpleMemStore.java|  5 ---
 .../gridtable/AggregationCacheSpillTest.java| 12 +--
 .../kylin/gridtable/SimpleGridTableTest.java|  9 +-
 .../storage/gtrecord/CubeSegmentScanner.java|  5 ---
 .../kylin/storage/gtrecord/ScannerWorker.java   | 15 +++--
 .../gtrecord/StorageResponseGTScatter.java  |  9 +-
 .../hbase/cube/HBaseScannerBenchmark.java   |  3 --
 .../storage/hbase/cube/SimpleHBaseStore.java|  5 ---
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 22 ++---
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java | 33 ++--
 .../hbase/cube/v2/HBaseReadonlyStore.java   |  5 ---
 .../endpoint/protobuf/CubeVisit.proto   |  2 +-
 .../hbase/steps/SandboxMetastoreCLI.java|  1 -
 22 files changed, 27 insertions(+), 157 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/a7a318c2/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/ConcurrentDiskStore.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/ConcurrentDiskStore.java
 
b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/ConcurrentDiskStore.java
index 5219ede..41d2dfb 100644
--- 
a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/ConcurrentDiskStore.java
+++ 
b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/ConcurrentDiskStore.java
@@ -265,11 +265,6 @@ public class ConcurrentDiskStore implements IGTStore, 
Closeable {
 return info;
 }
 
-@Override
-public long getScannedRowCount() {
-return count;
-}
-
 }
 
 private class Writer implements IGTWriter {
@@ -371,4 +366,4 @@ public class ConcurrentDiskStore implements IGTStore, 
Closeable {
 return "ConcurrentDiskStore@" + (info.getTableName() == null ? 
this.hashCode() : info.getTableName());
 }
 
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/a7a318c2/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
 
b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
index a74f0c0..e08844e 100644
--- 
a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
+++ 
b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
@@ -564,10 +564,5 @@ public class InMemCubeBuilder extends 
AbstractInMemCubeBuilder {
 public GTInfo getInfo() {
 return info;
 }
-
-@Override
-public long getScannedRowCount() {
-return 0L;
-}
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/a7a318c2/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/MemDiskStore.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/MemDiskStore.java 
b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/MemDiskStore.java
index 81403ab..a5471df 100644
--- 
a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/MemDiskStore.java
+++ 
b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/MemDiskStore.java
@@ -277,12 +277,6 @@ public class MemDiskStore implements IGTStore, Closeable {
 public GTInfo getInfo() {
 return info;
 }
-
-@Override
-public long getScannedRowCount() {
-return count;
-}
-
 }
 
 private class Writer implements IGTWriter {

kylin git commit: refactor

[billyliu]

Repository: kylin
Updated Branches:
  refs/heads/hive-load-refactor 0b66ef1d6 -> 877b538a5


refactor


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/877b538a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/877b538a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/877b538a

Branch: refs/heads/hive-load-refactor
Commit: 877b538a508f60ab72d97eaf7af6e39d55a4b4bd
Parents: 0b66ef1
Author: Billy Liu 
Authored: Tue Dec 27 10:22:50 2016 +0800
Committer: Billy Liu 
Committed: Tue Dec 27 10:22:50 2016 +0800

--
 .../kylin/rest/controller/TableController.java  | 76 ++--
 .../apache/kylin/rest/service/TableService.java | 72 +++
 webapp/app/js/controllers/sourceMeta.js | 16 ++---
 webapp/app/js/services/tables.js|  1 -
 4 files changed, 86 insertions(+), 79 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/877b538a/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
 
b/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
index 0debad7..964cd4c 100644
--- 
a/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
+++ 
b/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
@@ -24,12 +24,11 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
-import org.apache.kylin.common.util.JsonUtil;
 import org.apache.kylin.metadata.model.TableDesc;
 import org.apache.kylin.rest.exception.InternalErrorException;
+import org.apache.kylin.rest.exception.NotFoundException;
 import org.apache.kylin.rest.request.CardinalityRequest;
 import org.apache.kylin.rest.request.HiveTableRequest;
-import org.apache.kylin.rest.request.StreamingRequest;
 import org.apache.kylin.rest.service.TableService;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -66,14 +65,12 @@ public class TableController extends BasicController {
 @RequestMapping(value = "", method = { RequestMethod.GET })
 @ResponseBody
 public List getTableDesc(@RequestParam(value = "ext", required 
= false) boolean withExt, @RequestParam(value = "project", required = true) 
String project) throws IOException {
-List tables = null;
 try {
-tables = tableService.getTableDescByProject(project, withExt);
+return tableService.getTableDescByProject(project, withExt);
 } catch (IOException e) {
 logger.error("Failed to get Hive Tables", e);
 throw new InternalErrorException(e.getLocalizedMessage());
 }
-return tables;
 }
 
 /**
@@ -85,21 +82,28 @@ public class TableController extends BasicController {
 @RequestMapping(value = "/{tableName:.+}", method = { RequestMethod.GET })
 @ResponseBody
 public TableDesc getTableDesc(@PathVariable String tableName) {
-return tableService.getTableDescByName(tableName);
+TableDesc table = tableService.getTableDescByName(tableName, false);
+if (table == null)
+throw new NotFoundException("Could not find Hive table: " + 
tableName);
+return table;
 }
 
 @RequestMapping(value = "/{tables}/{project}", method = { 
RequestMethod.POST })
 @ResponseBody
 public Map loadHiveTables(@PathVariable String tables, 
@PathVariable String project, @RequestBody HiveTableRequest request) throws 
IOException {
 String submitter = 
SecurityContextHolder.getContext().getAuthentication().getName();
+Map result = new HashMap();
 String[] tableNames = tables.split(",");
-String[] loaded = tableService.loadHiveTablesToProject(tableNames, 
project);
-if (request.isCalculate()) {
-tableService.calculateCardinalityIfNotPresent(loaded, submitter);
+try {
+String[] loaded = tableService.loadHiveTablesToProject(tableNames, 
project);
+result.put("result.loaded", loaded);
+if (request.isCalculate()) {
+tableService.calculateCardinalityIfNotPresent(loaded, 
submitter);
+}
+} catch (Exception e) {
+logger.error("Failed to load Hive Table", e);
+throw new InternalErrorException(e.getLocalizedMessage());
 }
-Map result = new HashMap();
-result.put("result.loaded", loaded);
-result.put("result.unloaded", new String[] {});
 return result;
 }
 
@@ -109,29 +113,23 @@ public class TableController extends 

kylin git commit: refactor to TableService

[billyliu]

Repository: kylin
Updated Branches:
  refs/heads/hive-load-refactor [created] 0b66ef1d6


refactor to TableService


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0b66ef1d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0b66ef1d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0b66ef1d

Branch: refs/heads/hive-load-refactor
Commit: 0b66ef1d60e8efc1c29d7d8c247d3cdd7506cb58
Parents: a78b650
Author: Billy Liu 
Authored: Mon Dec 26 23:42:11 2016 +0800
Committer: Billy Liu 
Committed: Mon Dec 26 23:42:11 2016 +0800

--
 .../hive/ITHiveSourceTableLoaderTest.java   |   2 +-
 .../rest/controller/StreamingController.java|  22 +-
 .../kylin/rest/controller/TableController.java  | 199 ++--
 .../apache/kylin/rest/service/CubeService.java  |  96 --
 .../apache/kylin/rest/service/TableService.java | 307 +++
 .../source/hive/HiveSourceTableLoader.java  |   2 +-
 6 files changed, 333 insertions(+), 295 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/0b66ef1d/kylin-it/src/test/java/org/apache/kylin/source/hive/ITHiveSourceTableLoaderTest.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/source/hive/ITHiveSourceTableLoaderTest.java
 
b/kylin-it/src/test/java/org/apache/kylin/source/hive/ITHiveSourceTableLoaderTest.java
index c4f0777..7aff3ba 100644
--- 
a/kylin-it/src/test/java/org/apache/kylin/source/hive/ITHiveSourceTableLoaderTest.java
+++ 
b/kylin-it/src/test/java/org/apache/kylin/source/hive/ITHiveSourceTableLoaderTest.java
@@ -45,7 +45,7 @@ public class ITHiveSourceTableLoaderTest extends 
HBaseMetadataTestCase {
 public void test() throws IOException {
 KylinConfig config = getTestConfig();
 String[] toLoad = new String[] { "DEFAULT.TEST_KYLIN_FACT", 
"EDW.TEST_CAL_DT" };
-Set loaded = HiveSourceTableLoader.reloadHiveTables(toLoad, 
config);
+Set loaded = HiveSourceTableLoader.loadHiveTables(toLoad, 
config);
 
 assertTrue(loaded.size() == toLoad.length);
 for (String str : toLoad)

http://git-wip-us.apache.org/repos/asf/kylin/blob/0b66ef1d/server-base/src/main/java/org/apache/kylin/rest/controller/StreamingController.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/controller/StreamingController.java
 
b/server-base/src/main/java/org/apache/kylin/rest/controller/StreamingController.java
index e04ebc8..0ced9ad 100644
--- 
a/server-base/src/main/java/org/apache/kylin/rest/controller/StreamingController.java
+++ 
b/server-base/src/main/java/org/apache/kylin/rest/controller/StreamingController.java
@@ -23,10 +23,8 @@ import java.util.List;
 import java.util.UUID;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.JsonUtil;
 import org.apache.kylin.engine.mr.HadoopUtil;
-import org.apache.kylin.metadata.MetadataManager;
 import org.apache.kylin.metadata.model.TableDesc;
 import org.apache.kylin.metadata.streaming.StreamingConfig;
 import org.apache.kylin.rest.exception.BadRequestException;
@@ -34,9 +32,9 @@ import org.apache.kylin.rest.exception.ForbiddenException;
 import org.apache.kylin.rest.exception.InternalErrorException;
 import org.apache.kylin.rest.exception.NotFoundException;
 import org.apache.kylin.rest.request.StreamingRequest;
-import org.apache.kylin.rest.service.CubeService;
 import org.apache.kylin.rest.service.KafkaConfigService;
 import org.apache.kylin.rest.service.StreamingService;
+import org.apache.kylin.rest.service.TableService;
 import org.apache.kylin.source.kafka.config.KafkaConfig;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -69,7 +67,7 @@ public class StreamingController extends BasicController {
 @Autowired
 private KafkaConfigService kafkaConfigService;
 @Autowired
-private CubeService cubeMgmtService;
+private TableService tableService;
 
 @RequestMapping(value = "/getConfig", method = { RequestMethod.GET })
 @ResponseBody
@@ -113,10 +111,7 @@ public class StreamingController extends BasicController {
 boolean saveStreamingSuccess = false, saveKafkaSuccess = false;
 
 try {
-tableDesc.setUuid(UUID.randomUUID().toString());
-MetadataManager metaMgr = 
MetadataManager.getInstance(KylinConfig.getInstanceFromEnv());
-metaMgr.saveSourceTable(tableDesc);
-cubeMgmtService.syncTableToProject(new String[] { 
tableDesc.getIdentity() }, project);
+tableService.addStreamingTable(tableDesc, project);
 } catch (IOException e) {
 throw new 

[24/50] [abbrv] kylin git commit: refactor: rename MeasureDecoder to MeasureCodec

[liyang]

refactor: rename MeasureDecoder to MeasureCodec


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/fa5c9cb7
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/fa5c9cb7
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/fa5c9cb7

Branch: refs/heads/1.5.x-HBase1.x
Commit: fa5c9cb7851afff028f5a2d39d12fbbb32bd40e8
Parents: 4c9b67f
Author: Hongbin Ma 
Authored: Thu Sep 22 11:22:32 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Sep 22 18:50:35 2016 +0800

--
 .../kylin/gridtable/GTAggregateScanner.java |   8 +-
 .../metadata/measure/MeasureCodecTest.java  |   4 +-
 .../kylin/measure/BufferedMeasureCodec.java | 107 +++
 .../kylin/measure/BufferedMeasureEncoder.java   | 107 ---
 .../org/apache/kylin/measure/MeasureCodec.java  | 101 +
 .../apache/kylin/measure/MeasureDecoder.java|  97 -
 .../kylin/engine/mr/common/CuboidStatsUtil.java |   4 +-
 .../engine/mr/steps/BaseCuboidMapperBase.java   |   6 +-
 .../kylin/engine/mr/steps/CuboidReducer.java|   6 +-
 .../mr/steps/FactDistinctHiveColumnsMapper.java |   4 +-
 .../engine/mr/steps/InMemCuboidReducer.java |   6 +-
 .../kylin/engine/mr/steps/KVGTRecordWriter.java |   4 +-
 .../engine/mr/steps/MergeCuboidMapper.java  |   6 +-
 .../kylin/engine/mr/steps/CubeReducerTest.java  |   8 +-
 .../apache/kylin/engine/spark/SparkCubing.java  |   4 +-
 .../spark/cube/DefaultTupleConverter.java   |   4 +-
 .../cardinality/ColumnCardinalityMapper.java|   4 +-
 .../cardinality/ColumnCardinalityReducer.java   |   4 +-
 .../ColumnCardinalityReducerTest.java   |   4 +-
 .../observer/ObserverAggregators.java   |   6 +-
 .../coprocessor/endpoint/CubeVisitService.java  |   6 +-
 .../storage/hbase/steps/CubeHFileMapper.java|   6 +-
 .../storage/hbase/steps/KeyValueCreator.java|   6 +-
 .../storage/hbase/steps/RowValueDecoder.java|   6 +-
 .../hbase/steps/CubeHFileMapper2Test.java   |   6 +-
 .../hbase/steps/RowValueDecoderTest.java|   6 +-
 26 files changed, 267 insertions(+), 263 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/fa5c9cb7/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java 
b/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
index db38484..9bbcd10 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
@@ -40,7 +40,7 @@ import org.apache.kylin.common.util.ImmutableBitSet;
 import org.apache.kylin.common.util.MemoryBudgetController;
 import org.apache.kylin.common.util.Pair;
 import org.apache.kylin.common.util.MemoryBudgetController.MemoryWaterLevel;
-import org.apache.kylin.measure.BufferedMeasureEncoder;
+import org.apache.kylin.measure.BufferedMeasureCodec;
 import org.apache.kylin.measure.MeasureAggregator;
 import org.apache.kylin.measure.MeasureAggregators;
 import org.apache.kylin.metadata.datatype.DataType;
@@ -178,7 +178,7 @@ public class GTAggregateScanner implements IGTScanner {
 final List dumps;
 final int keyLength;
 final boolean[] compareMask;
-final BufferedMeasureEncoder measureCodec;
+final BufferedMeasureCodec measureCodec;
 
 final Comparator bytesComparator = new Comparator() {
 @Override
@@ -212,13 +212,13 @@ public class GTAggregateScanner implements IGTScanner {
 measureCodec = createMeasureCodec();
 }
 
-private BufferedMeasureEncoder createMeasureCodec() {
+private BufferedMeasureCodec createMeasureCodec() {
 DataType[] types = new DataType[metrics.trueBitCount()];
 for (int i = 0; i < types.length; i++) {
 types[i] = info.getColumnType(metrics.trueBitAt(i));
 }
 
-BufferedMeasureEncoder result = new BufferedMeasureEncoder(types);
+BufferedMeasureCodec result = new BufferedMeasureCodec(types);
 result.setBufferSize(info.getMaxColumnLength(metrics));
 return result;
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/fa5c9cb7/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java
--
diff --git 
a/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java
 
b/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java
index 1a7e4b3..64c8970 100644
--- 

[2/2] kylin git commit: refactor BuildCubeWithStream

[shaofengshi]

refactor BuildCubeWithStream

Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a08dd2e0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a08dd2e0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a08dd2e0

Branch: refs/heads/master
Commit: a08dd2e03900b321617647d1dbf1c4ee8b4b18c2
Parents: c9dcb07
Author: shaofengshi 
Authored: Mon Sep 19 20:18:35 2016 +0800
Committer: shaofengshi 
Committed: Mon Sep 19 22:16:06 2016 +0800

--
 .../java/org/apache/kylin/job/DeployUtil.java   |   7 +-
 .../kylin/provision/BuildCubeWithStream.java|  10 +-
 .../kylin/provision/BuildCubeWithStream2.java   | 145 +--
 3 files changed, 12 insertions(+), 150 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/a08dd2e0/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
--
diff --git a/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java 
b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
index 9b282e3..9e9df05 100644
--- a/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
+++ b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
@@ -187,6 +187,7 @@ public class DeployUtil {
 File tmpFile = File.createTempFile(factTableName, "csv");
 FileOutputStream out = new FileOutputStream(tmpFile);
 
+InputStream tempIn = null;
 try {
 if (store.exists(factTablePath)) {
 InputStream oldContent = 
store.getResource(factTablePath).inputStream;
@@ -194,13 +195,15 @@ public class DeployUtil {
 }
 IOUtils.copy(in, out);
 IOUtils.closeQuietly(in);
+IOUtils.closeQuietly(out);
 
 store.deleteResource(factTablePath);
-in = new FileInputStream(tmpFile);
-store.putResource(factTablePath, in, System.currentTimeMillis());
+tempIn = new FileInputStream(tmpFile);
+store.putResource(factTablePath, tempIn, 
System.currentTimeMillis());
 } finally {
 IOUtils.closeQuietly(out);
 IOUtils.closeQuietly(in);
+IOUtils.closeQuietly(tempIn);
 }
 
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/a08dd2e0/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java 
b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
index 6e5313f..bfe1d0a 100644
--- a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
+++ b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
@@ -62,10 +62,10 @@ public class BuildCubeWithStream {
 
 private static final Logger logger = 
LoggerFactory.getLogger(org.apache.kylin.provision.BuildCubeWithStream.class);
 
-private CubeManager cubeManager;
+protected CubeManager cubeManager;
 private DefaultScheduler scheduler;
 protected ExecutableManager jobService;
-private static final String cubeName = "test_streaming_table_cube";
+static final String cubeName = "test_streaming_table_cube";
 
 private KafkaConfig kafkaConfig;
 private MockKafka kafkaServer;
@@ -114,13 +114,13 @@ public class BuildCubeWithStream {
 Assert.assertEquals(topicName, topicMetadata.topic());
 }
 
-private void generateStreamData(long startTime, long endTime, int 
numberOfRecords) throws IOException {
+protected void generateStreamData(long startTime, long endTime, int 
numberOfRecords) throws IOException {
 Kafka10DataLoader dataLoader = new Kafka10DataLoader(kafkaConfig);
 DeployUtil.prepareTestDataForStreamingCube(startTime, endTime, 
numberOfRecords, cubeName, dataLoader);
 logger.info("Test data inserted into Kafka");
 }
 
-private void clearSegment(String cubeName) throws Exception {
+protected void clearSegment(String cubeName) throws Exception {
 CubeInstance cube = cubeManager.getCube(cubeName);
 // remove all existing segments
 CubeUpdate cubeBuilder = new CubeUpdate(cube);
@@ -187,7 +187,7 @@ public class BuildCubeWithStream {
 return job.getId();
 }
 
-private ExecutableState buildSegment(String cubeName, long startOffset, 
long endOffset) throws Exception {
+protected ExecutableState buildSegment(String cubeName, long startOffset, 
long endOffset) throws Exception {
 CubeSegment segment = 
cubeManager.appendSegment(cubeManager.getCube(cubeName), 0, 0, startOffset, 
endOffset);
 DefaultChainedExecutable job = 

[1/6] kylin git commit: refactor some streaming classes

[shaofengshi]

Repository: kylin
Updated Branches:
  refs/heads/master 2146f2b00 -> c67891d26


refactor some streaming classes


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a2b693c7
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a2b693c7
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a2b693c7

Branch: refs/heads/master
Commit: a2b693c7955b4e5c436f4f815cd5588da93f7e98
Parents: 2146f2b
Author: shaofengshi 
Authored: Thu Jun 23 10:34:52 2016 +0800
Committer: shaofengshi 
Committed: Wed Aug 10 10:09:22 2016 +0800

--
 .../src/test/java/org/apache/kylin/job/DeployUtil.java  |  2 +-
 .../org/apache/kylin/common/util/StreamingMessage.java  |  4 
 .../apache/kylin/source/kafka/KafkaStreamingInput.java  |  3 ++-
 .../org/apache/kylin/source/kafka/StreamingParser.java  |  3 ++-
 .../kylin/source/kafka/StringStreamingParser.java   | 12 
 .../kylin/source/kafka/TimedJsonStreamParser.java   | 12 
 .../kylin/source/kafka/diagnose/KafkaInputAnalyzer.java |  3 ++-
 .../org/apache/kylin/source/kafka/util/KafkaUtils.java  |  3 ++-
 8 files changed, 21 insertions(+), 21 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/a2b693c7/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
--
diff --git a/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java 
b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
index da97df3..6128770 100644
--- a/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
+++ b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
@@ -166,7 +166,7 @@ public class DeployUtil {
 TimedJsonStreamParser timedJsonStreamParser = new 
TimedJsonStreamParser(tableColumns, "formatTs=true");
 StringBuilder sb = new StringBuilder();
 for (String json : data) {
-List rowColumns = timedJsonStreamParser.parse(new 
MessageAndOffset(new Message(json.getBytes()), 0)).getData();
+List rowColumns = timedJsonStreamParser.parse((new 
MessageAndOffset(new Message(json.getBytes()), 
0)).message().payload()).getData();
 sb.append(StringUtils.join(rowColumns, ","));
 sb.append(System.getProperty("line.separator"));
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/a2b693c7/core-common/src/main/java/org/apache/kylin/common/util/StreamingMessage.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/util/StreamingMessage.java 
b/core-common/src/main/java/org/apache/kylin/common/util/StreamingMessage.java
index f327db2..53ab195 100644
--- 
a/core-common/src/main/java/org/apache/kylin/common/util/StreamingMessage.java
+++ 
b/core-common/src/main/java/org/apache/kylin/common/util/StreamingMessage.java
@@ -51,6 +51,10 @@ public class StreamingMessage {
 return offset;
 }
 
+public void setOffset(long offset) {
+this.offset = offset;
+}
+
 public final long getTimestamp() {
 return timestamp;
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/a2b693c7/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaStreamingInput.java
--
diff --git 
a/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaStreamingInput.java
 
b/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaStreamingInput.java
index 4a3c2a9..fe3fe0a 100644
--- 
a/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaStreamingInput.java
+++ 
b/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaStreamingInput.java
@@ -205,7 +205,8 @@ public class KafkaStreamingInput implements IStreamingInput 
{
 for (MessageAndOffset messageAndOffset : 
fetchResponse.messageSet(topic, partitionId)) {
 offset++;
 consumeMsgCount++;
-final StreamingMessage streamingMessage = 
streamingParser.parse(messageAndOffset);
+final StreamingMessage streamingMessage = 
streamingParser.parse(messageAndOffset.message().payload());
+streamingMessage.setOffset(messageAndOffset.offset());
 if (streamingParser.filter(streamingMessage)) {
 final long timestamp = 
streamingMessage.getTimestamp();
 if (timestamp >= timeRange.getFirst() && timestamp 
< timeRange.getSecond()) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/a2b693c7/source-kafka/src/main/java/org/apache/kylin/source/kafka/StreamingParser.java

[1/2] kylin git commit: refactor on sharding

[mahongbin]

Repository: kylin
Updated Branches:
  refs/heads/master aac4e7e49 -> 467226f40


refactor on sharding


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/49b37efd
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/49b37efd
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/49b37efd

Branch: refs/heads/master
Commit: 49b37efd1c741d0d5900c4d500912f604df41260
Parents: aac4e7e
Author: Hongbin Ma 
Authored: Mon Jul 25 13:35:51 2016 +0800
Committer: Hongbin Ma 
Committed: Tue Jul 26 13:19:45 2016 +0800

--
 .../java/org/apache/kylin/cube/CubeSegment.java | 38 
 .../org/apache/kylin/cube/kv/RowKeyEncoder.java |  2 +-
 .../kylin/engine/mr/common/CuboidShardUtil.java |  6 +--
 .../engine/mr/steps/CubingExecutableUtil.java   | 40 +
 .../engine/mr/steps/SaveStatisticsStep.java | 46 +---
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  2 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java |  2 +-
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  2 +-
 .../storage/hbase/steps/CreateHTableJob.java|  1 +
 9 files changed, 79 insertions(+), 60 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/49b37efd/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
index dca4381..febfa86 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
@@ -41,17 +41,20 @@ import org.apache.kylin.metadata.model.IJoinedFlatTableDesc;
 import org.apache.kylin.metadata.model.SegmentStatusEnum;
 import org.apache.kylin.metadata.model.TblColRef;
 import org.apache.kylin.metadata.realization.IRealization;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
 import com.fasterxml.jackson.annotation.JsonBackReference;
 import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
 
 @JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = 
Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = 
Visibility.NONE)
 public class CubeSegment implements Comparable, IBuildable {
+private static final Logger logger = 
LoggerFactory.getLogger(CubeSegment.class);
 
 @JsonBackReference
 private CubeInstance cubeInstance;
@@ -85,7 +88,7 @@ public class CubeSegment implements Comparable, 
IBuildable {
 private long createTimeUTC;
 @JsonProperty("cuboid_shard_nums")
 private Map cuboidShardNums = Maps.newHashMap();
-@JsonProperty("total_shards")
+@JsonProperty("total_shards") //it is only valid when all cuboids are 
squshed into some shards. like the HBASE_STORAGE case, otherwise it'll stay 0
 private int totalShards = 0;
 @JsonProperty("blackout_cuboids")
 private List blackoutCuboids = Lists.newArrayList();
@@ -489,8 +492,16 @@ public class CubeSegment implements 
Comparable, IBuildable {
 this.cuboidShardNums = newCuboidShards;
 }
 
-public int getTotalShards() {
-return totalShards;
+public int getTotalShards(long cuboidId) {
+if (totalShards > 0) {
+//shard squashed case
+logger.info("total shards for {} is {}", cuboidId, totalShards);
+return totalShards;
+} else {
+int ret = getCuboidShardNum(cuboidId);
+logger.info("total shards for {} is {}", cuboidId, ret);
+return ret;
+}
 }
 
 public void setTotalShards(int totalShards) {
@@ -498,12 +509,21 @@ public class CubeSegment implements 
Comparable, IBuildable {
 }
 
 public short getCuboidBaseShard(Long cuboidId) {
-Short ret = cuboidBaseShards.get(cuboidId);
-if (ret == null) {
-ret = ShardingHash.getShard(cuboidId, totalShards);
-cuboidBaseShards.put(cuboidId, ret);
+if (totalShards > 0) {
+//shard squashed case
+
+Short ret = cuboidBaseShards.get(cuboidId);
+if (ret == null) {
+ret = ShardingHash.getShard(cuboidId, totalShards);
+cuboidBaseShards.put(cuboidId, ret);
+}
+
+logger.info("base for cuboid {} is {}", cuboidId, ret);
+return ret;
+} else {
+

[46/50] kylin git commit: refactor & cleanup UserService

[shaofengshi]

refactor & cleanup UserService


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c7439a0a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c7439a0a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c7439a0a

Branch: refs/heads/stream_m1
Commit: c7439a0a4281d1633437f268fdb5d44fbcd6cde1
Parents: 4fd74fc
Author: Yang Li 
Authored: Fri Jun 24 08:10:39 2016 +0800
Committer: Yang Li 
Committed: Fri Jun 24 08:10:39 2016 +0800

--
 .../kylin/rest/controller/UserController.java   |  2 +-
 .../apache/kylin/rest/security/UserManager.java | 29 -
 .../apache/kylin/rest/service/UserService.java  | 33 +++-
 .../kylin/rest/service/UserServiceTest.java |  2 +-
 4 files changed, 20 insertions(+), 46 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/c7439a0a/server/src/main/java/org/apache/kylin/rest/controller/UserController.java
--
diff --git 
a/server/src/main/java/org/apache/kylin/rest/controller/UserController.java 
b/server/src/main/java/org/apache/kylin/rest/controller/UserController.java
index a7bc782..54af010 100644
--- a/server/src/main/java/org/apache/kylin/rest/controller/UserController.java
+++ b/server/src/main/java/org/apache/kylin/rest/controller/UserController.java
@@ -76,7 +76,7 @@ public class UserController {
 
 @RequestMapping(value = "/authentication/authorities", method = 
RequestMethod.GET, produces = "application/json")
 public List getAuthorities() {
-return userService.getUserAuthorities();
+return userService.listUserAuthorities();
 }
 
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/c7439a0a/server/src/main/java/org/apache/kylin/rest/security/UserManager.java
--
diff --git 
a/server/src/main/java/org/apache/kylin/rest/security/UserManager.java 
b/server/src/main/java/org/apache/kylin/rest/security/UserManager.java
deleted file mode 100644
index bbdbc2b..000
--- a/server/src/main/java/org/apache/kylin/rest/security/UserManager.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- * http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.rest.security;
-
-import java.util.List;
-
-import org.springframework.security.provisioning.UserDetailsManager;
-
-public interface UserManager extends UserDetailsManager {
-
-public List getUserAuthorities();
-
-}

http://git-wip-us.apache.org/repos/asf/kylin/blob/c7439a0a/server/src/main/java/org/apache/kylin/rest/service/UserService.java
--
diff --git 
a/server/src/main/java/org/apache/kylin/rest/service/UserService.java 
b/server/src/main/java/org/apache/kylin/rest/service/UserService.java
index a3b4293..d38a726 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/UserService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/UserService.java
@@ -38,13 +38,13 @@ import org.apache.hadoop.hbase.client.Scan;
 import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.common.util.Pair;
 import org.apache.kylin.rest.security.AclHBaseStorage;
-import org.apache.kylin.rest.security.UserManager;
 import org.apache.kylin.rest.util.Serializer;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.security.core.GrantedAuthority;
 import org.springframework.security.core.userdetails.User;
 import org.springframework.security.core.userdetails.UserDetails;
 import org.springframework.security.core.userdetails.UsernameNotFoundException;
+import org.springframework.security.provisioning.UserDetailsManager;
 import org.springframework.stereotype.Component;
 
 import com.fasterxml.jackson.core.JsonParseException;
@@ -52,11 +52,9 @@ import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.JsonMappingException;
 
 /**
- * @author xduo
- * 
  */
 

[50/50] kylin git commit: refactor some streaming classes

[shaofengshi]

refactor some streaming classes


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/32a08f8b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/32a08f8b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/32a08f8b

Branch: refs/heads/stream_m1
Commit: 32a08f8bf441f67af391947af944cc853028b274
Parents: 4ffe818
Author: shaofengshi 
Authored: Thu Jun 23 10:34:52 2016 +0800
Committer: shaofengshi 
Committed: Fri Jun 24 15:04:19 2016 +0800

--
 .../src/test/java/org/apache/kylin/job/DeployUtil.java|  2 +-
 .../org/apache/kylin/job/hadoop/invertedindex/IITest.java |  8 ++--
 .../org/apache/kylin/common/util/StreamingMessage.java|  4 
 .../apache/kylin/source/kafka/KafkaStreamingInput.java|  3 ++-
 .../org/apache/kylin/source/kafka/StreamingParser.java|  3 ++-
 .../apache/kylin/source/kafka/StringStreamingParser.java  | 10 --
 .../apache/kylin/source/kafka/TimedJsonStreamParser.java  |  9 -
 .../kylin/source/kafka/diagnose/KafkaInputAnalyzer.java   |  3 ++-
 .../org/apache/kylin/source/kafka/util/KafkaUtils.java|  3 ++-
 9 files changed, 23 insertions(+), 22 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/32a08f8b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
--
diff --git a/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java 
b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
index a0a9f88..d56dd64 100644
--- a/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
+++ b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
@@ -168,7 +168,7 @@ public class DeployUtil {
 TimedJsonStreamParser timedJsonStreamParser = new 
TimedJsonStreamParser(tableColumns, "formatTs=true");
 StringBuilder sb = new StringBuilder();
 for (String json : data) {
-List rowColumns = timedJsonStreamParser.parse(new 
MessageAndOffset(new Message(json.getBytes()), 0)).getData();
+List rowColumns = timedJsonStreamParser.parse((new 
MessageAndOffset(new Message(json.getBytes()), 
0)).message().payload()).getData();
 sb.append(StringUtils.join(rowColumns, ","));
 sb.append(System.getProperty("line.separator"));
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/32a08f8b/assembly/src/test/java/org/apache/kylin/job/hadoop/invertedindex/IITest.java
--
diff --git 
a/assembly/src/test/java/org/apache/kylin/job/hadoop/invertedindex/IITest.java 
b/assembly/src/test/java/org/apache/kylin/job/hadoop/invertedindex/IITest.java
index da25143..c34ce55 100644
--- 
a/assembly/src/test/java/org/apache/kylin/job/hadoop/invertedindex/IITest.java
+++ 
b/assembly/src/test/java/org/apache/kylin/job/hadoop/invertedindex/IITest.java
@@ -38,11 +38,7 @@ import org.apache.commons.lang.NotImplementedException;
 import org.apache.hadoop.hbase.Cell;
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.regionserver.RegionScanner;
-import org.apache.kylin.common.util.FIFOIterable;
-import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.apache.kylin.common.util.Pair;
-import org.apache.kylin.common.util.StreamingBatch;
-import org.apache.kylin.common.util.StreamingMessage;
+import org.apache.kylin.common.util.*;
 import org.apache.kylin.invertedindex.IIInstance;
 import org.apache.kylin.invertedindex.IIManager;
 import org.apache.kylin.invertedindex.IISegment;
@@ -114,7 +110,7 @@ public class IITest extends LocalFileMetadataTestCase {
 @Nullable
 @Override
 public StreamingMessage apply(@Nullable MessageAndOffset input) {
-return parser.parse(input);
+return parser.parse(input.message().payload());
 }
 });
 StreamingBatch batch = new StreamingBatch(streamingMessages, 
Pair.newPair(0L, System.currentTimeMillis()));

http://git-wip-us.apache.org/repos/asf/kylin/blob/32a08f8b/core-common/src/main/java/org/apache/kylin/common/util/StreamingMessage.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/util/StreamingMessage.java 
b/core-common/src/main/java/org/apache/kylin/common/util/StreamingMessage.java
index f327db2..53ab195 100644
--- 
a/core-common/src/main/java/org/apache/kylin/common/util/StreamingMessage.java
+++ 
b/core-common/src/main/java/org/apache/kylin/common/util/StreamingMessage.java
@@ -51,6 +51,10 @@ public class StreamingMessage {
 return offset;
 }
 
+public void setOffset(long offset) {
+this.offset = offset;
+}
+
 public 

[29/50] kylin git commit: refactor: remove maskForEqualHashComp from GTRecord

[shaofengshi]

refactor: remove maskForEqualHashComp from GTRecord


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/64b5f86f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/64b5f86f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/64b5f86f

Branch: refs/heads/stream_m1
Commit: 64b5f86fec6fce643cbd0f0f5a966ddc919dfbed
Parents: ff1c813
Author: Hongbin Ma 
Authored: Tue Jun 21 14:50:27 2016 +0800
Committer: Hongbin Ma 
Committed: Wed Jun 22 14:00:15 2016 +0800

--
 .../org/apache/kylin/common/util/BasicTest.java |  3 +
 .../kylin/gridtable/FuzzyKeyGTRecord.java   | 37 
 .../org/apache/kylin/gridtable/GTRecord.java| 60 ++--
 .../org/apache/kylin/gridtable/GTScanRange.java | 16 +++---
 .../kylin/gridtable/GTScanRangePlanner.java | 16 +++---
 .../apache/kylin/gridtable/GTScanRequest.java   | 21 +--
 .../storage/hbase/cube/v2/CubeHBaseRPC.java |  7 ++-
 7 files changed, 91 insertions(+), 69 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/64b5f86f/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java
--
diff --git 
a/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java 
b/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java
index f1f5aa4..81afafe 100644
--- a/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java
+++ b/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java
@@ -79,6 +79,9 @@ public class BasicTest {
 
 @Test
 public void testxx() throws InterruptedException {
+byte[] space = new byte[100];
+ByteBuffer buffer = ByteBuffer.wrap(space, 10, 20);
+buffer.put((byte) 1);
 }
 
 @Test

http://git-wip-us.apache.org/repos/asf/kylin/blob/64b5f86f/core-cube/src/main/java/org/apache/kylin/gridtable/FuzzyKeyGTRecord.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/gridtable/FuzzyKeyGTRecord.java 
b/core-cube/src/main/java/org/apache/kylin/gridtable/FuzzyKeyGTRecord.java
new file mode 100644
index 000..acffb4d
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/FuzzyKeyGTRecord.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *  
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *  
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.gridtable;
+
+import org.apache.kylin.common.util.ByteArray;
+import org.apache.kylin.common.util.ImmutableBitSet;
+
+public class FuzzyKeyGTRecord extends GTRecord {
+final ImmutableBitSet maskForEqualHashComp;
+
+public FuzzyKeyGTRecord(GTInfo info, ByteArray[] cols, ImmutableBitSet 
maskForEqualHashComp) {
+super(info, cols);
+this.maskForEqualHashComp = maskForEqualHashComp;
+}
+
+public FuzzyKeyGTRecord(GTInfo info,ImmutableBitSet maskForEqualHashComp)
+{
+super(info);
+this.maskForEqualHashComp = maskForEqualHashComp;
+}
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/64b5f86f/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java 
b/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java
index f3cfc6a..b676693 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java
@@ -31,39 +31,27 @@ public class GTRecord implements Comparable {
 
 final transient GTInfo info;
 final ByteArray[] cols;
-final ImmutableBitSet maskForEqualHashComp;
 
-public GTRecord(GTInfo info, ImmutableBitSet maskForEqualHashComp, 
ByteArray[] cols) {
+public GTRecord(GTInfo info, ByteArray[] cols) {
 this.info = info;
 this.cols = cols;
-this.maskForEqualHashComp = maskForEqualHashComp;
 }
 
-public GTRecord(GTInfo info, 

[36/50] kylin git commit: refactor: subclass of AbstractExecutable change to use own logger

[shaofengshi]

refactor: subclass of AbstractExecutable change to use own logger


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/94567c15
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/94567c15
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/94567c15

Branch: refs/heads/stream_m1
Commit: 94567c150e94d0f6499bb26621869c83d117c5a4
Parents: bfee5ed
Author: Hongbin Ma 
Authored: Wed Jun 22 17:51:55 2016 +0800
Committer: Hongbin Ma 
Committed: Wed Jun 22 17:52:03 2016 +0800

--
 .../kylin/job/common/ShellExecutable.java   |  4 
 .../kylin/job/execution/AbstractExecutable.java |  2 +-
 .../org/apache/kylin/engine/mr/CubingJob.java   |  4 
 .../engine/mr/common/HadoopShellExecutable.java |  4 
 .../UpdateIIInfoAfterBuildStep.java |  4 
 .../engine/mr/steps/MergeDictionaryStep.java|  5 
 .../engine/mr/steps/MergeStatisticsStep.java|  4 
 .../engine/mr/steps/SaveStatisticsStep.java |  4 
 .../mr/steps/UpdateCubeInfoAfterBuildStep.java  |  4 
 .../mr/steps/UpdateCubeInfoAfterMergeStep.java  |  4 
 .../kylin/engine/spark/SparkExecutable.java |  4 
 .../source/hive/CreateFlatHiveTableStep.java|  4 
 .../apache/kylin/source/hive/HiveMRInput.java   | 25 +++-
 .../apache/kylin/source/hive/HqlExecutable.java |  4 
 .../steps/HDFSPathGarbageCollectionStep.java|  5 
 15 files changed, 69 insertions(+), 12 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/94567c15/core-job/src/main/java/org/apache/kylin/job/common/ShellExecutable.java
--
diff --git 
a/core-job/src/main/java/org/apache/kylin/job/common/ShellExecutable.java 
b/core-job/src/main/java/org/apache/kylin/job/common/ShellExecutable.java
index 74f54a1..209d14e 100644
--- a/core-job/src/main/java/org/apache/kylin/job/common/ShellExecutable.java
+++ b/core-job/src/main/java/org/apache/kylin/job/common/ShellExecutable.java
@@ -32,11 +32,15 @@ import org.apache.kylin.job.execution.ExecutableContext;
 import org.apache.kylin.job.execution.ExecuteResult;
 
 import com.google.common.collect.Maps;
+import org.slf4j.LoggerFactory;
 
 /**
  */
 public class ShellExecutable extends AbstractExecutable {
 
+private static final org.slf4j.Logger logger = 
LoggerFactory.getLogger(ShellExecutable.class);
+
+
 private static final String CMD = "cmd";
 
 public ShellExecutable() {

http://git-wip-us.apache.org/repos/asf/kylin/blob/94567c15/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
--
diff --git 
a/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java 
b/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
index 395676b..778763c 100644
--- 
a/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
+++ 
b/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
@@ -49,7 +49,7 @@ public abstract class AbstractExecutable implements 
Executable, Idempotent {
 protected static final String START_TIME = "startTime";
 protected static final String END_TIME = "endTime";
 
-protected static final Logger logger = 
LoggerFactory.getLogger(AbstractExecutable.class);
+private static final Logger logger = 
LoggerFactory.getLogger(AbstractExecutable.class);
 protected int retry = 0;
 
 private String name;

http://git-wip-us.apache.org/repos/asf/kylin/blob/94567c15/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java
--
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java
index 70b6132..bbd0fc5 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java
@@ -42,11 +42,15 @@ import org.apache.kylin.job.execution.ExecuteResult;
 import org.apache.kylin.job.execution.Output;
 import org.apache.kylin.metadata.project.ProjectInstance;
 import org.apache.kylin.metadata.project.ProjectManager;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  */
 public class CubingJob extends DefaultChainedExecutable {
 
+private static final Logger logger = 
LoggerFactory.getLogger(CubingJob.class);
+
 public enum AlgorithmEnum {
 LAYER, INMEM
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/94567c15/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopShellExecutable.java

[30/50] kylin git commit: refactor: remove FuzzyKeyGTRecord

[shaofengshi]

refactor: remove FuzzyKeyGTRecord


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ea244504
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ea244504
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ea244504

Branch: refs/heads/stream_m1
Commit: ea244504e5fa66ef14923f6665bbdbafcd8c28e2
Parents: 64b5f86
Author: Hongbin Ma 
Authored: Tue Jun 21 14:56:50 2016 +0800
Committer: Hongbin Ma 
Committed: Wed Jun 22 14:00:15 2016 +0800

--
 .../kylin/gridtable/FuzzyKeyGTRecord.java   | 37 
 .../org/apache/kylin/gridtable/GTScanRange.java | 12 +++
 .../kylin/gridtable/GTScanRangePlanner.java | 18 +-
 .../apache/kylin/gridtable/GTScanRequest.java   | 18 +++---
 .../storage/hbase/cube/v2/CubeHBaseRPC.java |  7 ++--
 5 files changed, 22 insertions(+), 70 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/ea244504/core-cube/src/main/java/org/apache/kylin/gridtable/FuzzyKeyGTRecord.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/gridtable/FuzzyKeyGTRecord.java 
b/core-cube/src/main/java/org/apache/kylin/gridtable/FuzzyKeyGTRecord.java
deleted file mode 100644
index acffb4d..000
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/FuzzyKeyGTRecord.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *  
- * http://www.apache.org/licenses/LICENSE-2.0
- *  
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.gridtable;
-
-import org.apache.kylin.common.util.ByteArray;
-import org.apache.kylin.common.util.ImmutableBitSet;
-
-public class FuzzyKeyGTRecord extends GTRecord {
-final ImmutableBitSet maskForEqualHashComp;
-
-public FuzzyKeyGTRecord(GTInfo info, ByteArray[] cols, ImmutableBitSet 
maskForEqualHashComp) {
-super(info, cols);
-this.maskForEqualHashComp = maskForEqualHashComp;
-}
-
-public FuzzyKeyGTRecord(GTInfo info,ImmutableBitSet maskForEqualHashComp)
-{
-super(info);
-this.maskForEqualHashComp = maskForEqualHashComp;
-}
-}

http://git-wip-us.apache.org/repos/asf/kylin/blob/ea244504/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRange.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRange.java 
b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRange.java
index 433626e..0cffcd9 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRange.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRange.java
@@ -27,25 +27,25 @@ public class GTScanRange {
 
 final public GTRecord pkStart; // inclusive, record must not be null, 
col[pk].array() can be null to mean unbounded
 final public GTRecord pkEnd; // inclusive, record must not be null, 
col[pk].array() can be null to mean unbounded
-final public List fuzzyKeys; // partial matching primary 
keys
+final public List fuzzyKeys; // partial matching primary keys
 
 public GTScanRange(GTRecord pkStart, GTRecord pkEnd) {
 this(pkStart, pkEnd, null);
 }
 
-public GTScanRange(GTRecord pkStart, GTRecord pkEnd, 
List fuzzyKeys) {
+public GTScanRange(GTRecord pkStart, GTRecord pkEnd, List 
fuzzyKeys) {
 GTInfo info = pkStart.info;
 assert info == pkEnd.info;
 
 this.pkStart = pkStart;
 this.pkEnd = pkEnd;
-this.fuzzyKeys = fuzzyKeys == null ? Collections. 
emptyList() : fuzzyKeys;
+this.fuzzyKeys = fuzzyKeys == null ? Collections. 
emptyList() : fuzzyKeys;
 }
 
 public GTScanRange replaceGTInfo(final GTInfo gtInfo) {
-List newFuzzyKeys = Lists.newArrayList();
-for (FuzzyKeyGTRecord input : fuzzyKeys) {
-newFuzzyKeys.add(new FuzzyKeyGTRecord(gtInfo, input.cols, 
input.maskForEqualHashComp));
+List newFuzzyKeys = Lists.newArrayList();
+for (GTRecord input : fuzzyKeys) {
+newFuzzyKeys.add(new 

[45/50] kylin git commit: refactor about default gtstore

[shaofengshi]

refactor about default gtstore


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4fd74fc6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4fd74fc6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4fd74fc6

Branch: refs/heads/stream_m1
Commit: 4fd74fc6a7d7c9bcccab38b57da76f2b983cf5ef
Parents: 62ae3cb
Author: Hongbin Ma 
Authored: Thu Jun 23 16:37:09 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Jun 23 16:37:09 2016 +0800

--
 .../apache/kylin/gridtable/ScannerWorker.java   |  11 +-
 .../storage/gtrecord/CubeSegmentScanner.java| 100 +
 .../storage/gtrecord/CubeTupleConverter.java| 270 +
 .../gtrecord/GTCubeStorageQueryBase.java| 377 +++
 .../gtrecord/SequentialCubeTupleIterator.java   | 210 +++
 .../hbase/cube/v2/CubeSegmentScanner.java   | 100 -
 .../storage/hbase/cube/v2/CubeStorageQuery.java | 349 +
 .../hbase/cube/v2/CubeTupleConverter.java   | 270 -
 .../cube/v2/SequentialCubeTupleIterator.java| 210 ---
 9 files changed, 968 insertions(+), 929 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/4fd74fc6/core-cube/src/main/java/org/apache/kylin/gridtable/ScannerWorker.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/gridtable/ScannerWorker.java 
b/core-cube/src/main/java/org/apache/kylin/gridtable/ScannerWorker.java
index 1ac3b02..586a584 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/ScannerWorker.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/ScannerWorker.java
@@ -24,8 +24,6 @@ import java.io.IOException;
 import java.lang.reflect.InvocationTargetException;
 import java.util.Iterator;
 
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.debug.BackdoorToggles;
 import org.apache.kylin.cube.CubeSegment;
 import org.apache.kylin.cube.cuboid.Cuboid;
 import org.slf4j.Logger;
@@ -36,7 +34,7 @@ public class ScannerWorker {
 private static final Logger logger = 
LoggerFactory.getLogger(ScannerWorker.class);
 private IGTScanner internal = null;
 
-public ScannerWorker(CubeSegment cubeSeg, Cuboid cuboid, GTScanRequest 
scanRequest) {
+public ScannerWorker(CubeSegment cubeSeg, Cuboid cuboid, GTScanRequest 
scanRequest, String gtStorage) {
 if (scanRequest == null) {
 logger.info("Segment {} will be skipped", cubeSeg);
 internal = new EmptyGTScanner(0);
@@ -46,12 +44,7 @@ public class ScannerWorker {
 final GTInfo info = scanRequest.getInfo();
 
 try {
-IGTStorage rpc;
-if 
("scan".equalsIgnoreCase(BackdoorToggles.getHbaseCubeQueryProtocol())) {
-rpc = (IGTStorage) 
Class.forName("org.apache.kylin.storage.hbase.cube.v2.CubeHBaseScanRPC").getConstructor(CubeSegment.class,
 Cuboid.class, GTInfo.class).newInstance(cubeSeg, cuboid, info); // for local 
debug
-} else {
-rpc = (IGTStorage) 
Class.forName(KylinConfig.getInstanceFromEnv().getDefaultIGTStorage()).getConstructor(CubeSegment.class,
 Cuboid.class, GTInfo.class).newInstance(cubeSeg, cuboid, info); // default 
behavior
-}
+IGTStorage rpc = (IGTStorage) 
Class.forName(gtStorage).getConstructor(CubeSegment.class, Cuboid.class, 
GTInfo.class).newInstance(cubeSeg, cuboid, info); // default behavior
 internal = rpc.getGTScanner(scanRequest);
 } catch (IOException | InstantiationException | 
InvocationTargetException | IllegalAccessException | ClassNotFoundException | 
NoSuchMethodException e) {
 throw new RuntimeException("error", e);

http://git-wip-us.apache.org/repos/asf/kylin/blob/4fd74fc6/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
--
diff --git 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
new file mode 100644
index 000..c12159d
--- /dev/null
+++ 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *  
+ * 

[28/50] kylin git commit: refactor: minor changes

[shaofengshi]

refactor: minor changes


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b361d8aa
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b361d8aa
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b361d8aa

Branch: refs/heads/stream_m1
Commit: b361d8aaa2b1b5d0d86107bedb9f34397be000a5
Parents: ea24450
Author: Hongbin Ma 
Authored: Wed Jun 22 13:59:58 2016 +0800
Committer: Hongbin Ma 
Committed: Wed Jun 22 14:00:15 2016 +0800

--
 .../org/apache/kylin/cube/kv/RowKeyEncoder.java | 21 ++-
 .../org/apache/kylin/gridtable/GTRecord.java| 17 +-
 .../gridtable/memstore/GTSimpleMemStore.java| 20 +-
 .../kylin/gridtable/DictGridTableTest.java  | 64 ++--
 .../coprocessor/endpoint/CubeVisitService.java  |  2 +-
 5 files changed, 70 insertions(+), 54 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/b361d8aa/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoder.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoder.java 
b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoder.java
index db704dc..ebcbadd 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoder.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoder.java
@@ -94,12 +94,31 @@ public class RowKeyEncoder extends AbstractRowKeyEncoder {
 @Override
 public void encode(GTRecord record, ImmutableBitSet keyColumns, byte[] 
buf) {
 ByteArray byteArray = new ByteArray(buf, getHeaderLength(), 0);
-record.exportColumns(keyColumns, byteArray, defaultValue());
+
+encodeDims(record, keyColumns, byteArray, defaultValue());
 
 //fill shard and cuboid
 fillHeader(buf);
 }
 
+//ByteArray representing dimension does not have extra header
+public void encodeDims(GTRecord record, ImmutableBitSet selectedCols, 
ByteArray buf, byte defaultValue) {
+int pos = 0;
+for (int i = 0; i < selectedCols.trueBitCount(); i++) {
+int c = selectedCols.trueBitAt(i);
+ByteArray columnC = record.get(c);
+if (columnC.array() != null) {
+System.arraycopy(record.get(c).array(), columnC.offset(), 
buf.array(), buf.offset() + pos, columnC.length());
+pos += columnC.length();
+} else {
+int maxLength = 
record.getInfo().getCodeSystem().maxCodeLength(c);
+Arrays.fill(buf.array(), buf.offset() + pos, buf.offset() + 
pos + maxLength, defaultValue);
+pos += maxLength;
+}
+}
+buf.setLength(pos);
+}
+
 @Override
 public void encode(ByteArray bodyBytes, ByteArray outputBuf) {
 Preconditions.checkState(bodyBytes.length() == bodyLength);

http://git-wip-us.apache.org/repos/asf/kylin/blob/b361d8aa/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java 
b/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java
index b676693..276f66e 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java
@@ -235,22 +235,7 @@ public class GTRecord implements Comparable {
 buf.setLength(pos);
 }
 
-/** write data to given buffer, like serialize, use defaultValue when 
required column is not set*/
-public void exportColumns(ImmutableBitSet selectedCols, ByteArray buf, 
byte defaultValue) {
-int pos = 0;
-for (int i = 0; i < selectedCols.trueBitCount(); i++) {
-int c = selectedCols.trueBitAt(i);
-if (cols[c].array() != null) {
-System.arraycopy(cols[c].array(), cols[c].offset(), 
buf.array(), buf.offset() + pos, cols[c].length());
-pos += cols[c].length();
-} else {
-int maxLength = info.codeSystem.maxCodeLength(c);
-Arrays.fill(buf.array(), buf.offset() + pos, buf.offset() + 
pos + maxLength, defaultValue);
-pos += maxLength;
-}
-}
-buf.setLength(pos);
-}
+   
 
 /** write data to given buffer, like serialize */
 public void exportColumns(ImmutableBitSet selectedCols, ByteBuffer buf) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/b361d8aa/core-cube/src/main/java/org/apache/kylin/gridtable/memstore/GTSimpleMemStore.java
--
diff --git 

kylin git commit: refactor & cleanup UserService

[liyang]

Repository: kylin
Updated Branches:
  refs/heads/master 4fd74fc6a -> c7439a0a4


refactor & cleanup UserService


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c7439a0a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c7439a0a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c7439a0a

Branch: refs/heads/master
Commit: c7439a0a4281d1633437f268fdb5d44fbcd6cde1
Parents: 4fd74fc
Author: Yang Li 
Authored: Fri Jun 24 08:10:39 2016 +0800
Committer: Yang Li 
Committed: Fri Jun 24 08:10:39 2016 +0800

--
 .../kylin/rest/controller/UserController.java   |  2 +-
 .../apache/kylin/rest/security/UserManager.java | 29 -
 .../apache/kylin/rest/service/UserService.java  | 33 +++-
 .../kylin/rest/service/UserServiceTest.java |  2 +-
 4 files changed, 20 insertions(+), 46 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/c7439a0a/server/src/main/java/org/apache/kylin/rest/controller/UserController.java
--
diff --git 
a/server/src/main/java/org/apache/kylin/rest/controller/UserController.java 
b/server/src/main/java/org/apache/kylin/rest/controller/UserController.java
index a7bc782..54af010 100644
--- a/server/src/main/java/org/apache/kylin/rest/controller/UserController.java
+++ b/server/src/main/java/org/apache/kylin/rest/controller/UserController.java
@@ -76,7 +76,7 @@ public class UserController {
 
 @RequestMapping(value = "/authentication/authorities", method = 
RequestMethod.GET, produces = "application/json")
 public List getAuthorities() {
-return userService.getUserAuthorities();
+return userService.listUserAuthorities();
 }
 
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/c7439a0a/server/src/main/java/org/apache/kylin/rest/security/UserManager.java
--
diff --git 
a/server/src/main/java/org/apache/kylin/rest/security/UserManager.java 
b/server/src/main/java/org/apache/kylin/rest/security/UserManager.java
deleted file mode 100644
index bbdbc2b..000
--- a/server/src/main/java/org/apache/kylin/rest/security/UserManager.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- * http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.rest.security;
-
-import java.util.List;
-
-import org.springframework.security.provisioning.UserDetailsManager;
-
-public interface UserManager extends UserDetailsManager {
-
-public List getUserAuthorities();
-
-}

http://git-wip-us.apache.org/repos/asf/kylin/blob/c7439a0a/server/src/main/java/org/apache/kylin/rest/service/UserService.java
--
diff --git 
a/server/src/main/java/org/apache/kylin/rest/service/UserService.java 
b/server/src/main/java/org/apache/kylin/rest/service/UserService.java
index a3b4293..d38a726 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/UserService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/UserService.java
@@ -38,13 +38,13 @@ import org.apache.hadoop.hbase.client.Scan;
 import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.common.util.Pair;
 import org.apache.kylin.rest.security.AclHBaseStorage;
-import org.apache.kylin.rest.security.UserManager;
 import org.apache.kylin.rest.util.Serializer;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.security.core.GrantedAuthority;
 import org.springframework.security.core.userdetails.User;
 import org.springframework.security.core.userdetails.UserDetails;
 import org.springframework.security.core.userdetails.UsernameNotFoundException;
+import org.springframework.security.provisioning.UserDetailsManager;
 import org.springframework.stereotype.Component;
 
 import com.fasterxml.jackson.core.JsonParseException;
@@ -52,11 +52,9 @@ import com.fasterxml.jackson.core.JsonProcessingException;
 import 

kylin git commit: refactor about default gtstore

[mahongbin]

Repository: kylin
Updated Branches:
  refs/heads/master 62ae3cb16 -> 4fd74fc6a


refactor about default gtstore


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4fd74fc6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4fd74fc6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4fd74fc6

Branch: refs/heads/master
Commit: 4fd74fc6a7d7c9bcccab38b57da76f2b983cf5ef
Parents: 62ae3cb
Author: Hongbin Ma 
Authored: Thu Jun 23 16:37:09 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Jun 23 16:37:09 2016 +0800

--
 .../apache/kylin/gridtable/ScannerWorker.java   |  11 +-
 .../storage/gtrecord/CubeSegmentScanner.java| 100 +
 .../storage/gtrecord/CubeTupleConverter.java| 270 +
 .../gtrecord/GTCubeStorageQueryBase.java| 377 +++
 .../gtrecord/SequentialCubeTupleIterator.java   | 210 +++
 .../hbase/cube/v2/CubeSegmentScanner.java   | 100 -
 .../storage/hbase/cube/v2/CubeStorageQuery.java | 349 +
 .../hbase/cube/v2/CubeTupleConverter.java   | 270 -
 .../cube/v2/SequentialCubeTupleIterator.java| 210 ---
 9 files changed, 968 insertions(+), 929 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/4fd74fc6/core-cube/src/main/java/org/apache/kylin/gridtable/ScannerWorker.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/gridtable/ScannerWorker.java 
b/core-cube/src/main/java/org/apache/kylin/gridtable/ScannerWorker.java
index 1ac3b02..586a584 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/ScannerWorker.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/ScannerWorker.java
@@ -24,8 +24,6 @@ import java.io.IOException;
 import java.lang.reflect.InvocationTargetException;
 import java.util.Iterator;
 
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.debug.BackdoorToggles;
 import org.apache.kylin.cube.CubeSegment;
 import org.apache.kylin.cube.cuboid.Cuboid;
 import org.slf4j.Logger;
@@ -36,7 +34,7 @@ public class ScannerWorker {
 private static final Logger logger = 
LoggerFactory.getLogger(ScannerWorker.class);
 private IGTScanner internal = null;
 
-public ScannerWorker(CubeSegment cubeSeg, Cuboid cuboid, GTScanRequest 
scanRequest) {
+public ScannerWorker(CubeSegment cubeSeg, Cuboid cuboid, GTScanRequest 
scanRequest, String gtStorage) {
 if (scanRequest == null) {
 logger.info("Segment {} will be skipped", cubeSeg);
 internal = new EmptyGTScanner(0);
@@ -46,12 +44,7 @@ public class ScannerWorker {
 final GTInfo info = scanRequest.getInfo();
 
 try {
-IGTStorage rpc;
-if 
("scan".equalsIgnoreCase(BackdoorToggles.getHbaseCubeQueryProtocol())) {
-rpc = (IGTStorage) 
Class.forName("org.apache.kylin.storage.hbase.cube.v2.CubeHBaseScanRPC").getConstructor(CubeSegment.class,
 Cuboid.class, GTInfo.class).newInstance(cubeSeg, cuboid, info); // for local 
debug
-} else {
-rpc = (IGTStorage) 
Class.forName(KylinConfig.getInstanceFromEnv().getDefaultIGTStorage()).getConstructor(CubeSegment.class,
 Cuboid.class, GTInfo.class).newInstance(cubeSeg, cuboid, info); // default 
behavior
-}
+IGTStorage rpc = (IGTStorage) 
Class.forName(gtStorage).getConstructor(CubeSegment.class, Cuboid.class, 
GTInfo.class).newInstance(cubeSeg, cuboid, info); // default behavior
 internal = rpc.getGTScanner(scanRequest);
 } catch (IOException | InstantiationException | 
InvocationTargetException | IllegalAccessException | ClassNotFoundException | 
NoSuchMethodException e) {
 throw new RuntimeException("error", e);

http://git-wip-us.apache.org/repos/asf/kylin/blob/4fd74fc6/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
--
diff --git 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
new file mode 100644
index 000..c12159d
--- /dev/null
+++ 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. 

kylin git commit: refactor some streaming classes

[shaofengshi]

Repository: kylin
Updated Branches:
  refs/heads/stream_m1 [created] 3202cb259


refactor some streaming classes


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3202cb25
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3202cb25
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3202cb25

Branch: refs/heads/stream_m1
Commit: 3202cb259a1cf85deec368785bd6030a23f0e408
Parents: 5fee547
Author: shaofengshi 
Authored: Thu Jun 23 10:34:52 2016 +0800
Committer: shaofengshi 
Committed: Thu Jun 23 10:34:52 2016 +0800

--
 .../src/test/java/org/apache/kylin/job/DeployUtil.java|  2 +-
 .../org/apache/kylin/job/hadoop/invertedindex/IITest.java |  8 ++--
 .../org/apache/kylin/common/util/StreamingMessage.java|  4 
 .../apache/kylin/source/kafka/KafkaStreamingInput.java|  3 ++-
 .../org/apache/kylin/source/kafka/StreamingParser.java|  3 ++-
 .../apache/kylin/source/kafka/StringStreamingParser.java  | 10 --
 .../apache/kylin/source/kafka/TimedJsonStreamParser.java  |  9 -
 .../kylin/source/kafka/diagnose/KafkaInputAnalyzer.java   |  3 ++-
 .../org/apache/kylin/source/kafka/util/KafkaUtils.java|  3 ++-
 9 files changed, 23 insertions(+), 22 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/3202cb25/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
--
diff --git a/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java 
b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
index a0a9f88..d56dd64 100644
--- a/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
+++ b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
@@ -168,7 +168,7 @@ public class DeployUtil {
 TimedJsonStreamParser timedJsonStreamParser = new 
TimedJsonStreamParser(tableColumns, "formatTs=true");
 StringBuilder sb = new StringBuilder();
 for (String json : data) {
-List rowColumns = timedJsonStreamParser.parse(new 
MessageAndOffset(new Message(json.getBytes()), 0)).getData();
+List rowColumns = timedJsonStreamParser.parse((new 
MessageAndOffset(new Message(json.getBytes()), 
0)).message().payload()).getData();
 sb.append(StringUtils.join(rowColumns, ","));
 sb.append(System.getProperty("line.separator"));
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/3202cb25/assembly/src/test/java/org/apache/kylin/job/hadoop/invertedindex/IITest.java
--
diff --git 
a/assembly/src/test/java/org/apache/kylin/job/hadoop/invertedindex/IITest.java 
b/assembly/src/test/java/org/apache/kylin/job/hadoop/invertedindex/IITest.java
index da25143..c34ce55 100644
--- 
a/assembly/src/test/java/org/apache/kylin/job/hadoop/invertedindex/IITest.java
+++ 
b/assembly/src/test/java/org/apache/kylin/job/hadoop/invertedindex/IITest.java
@@ -38,11 +38,7 @@ import org.apache.commons.lang.NotImplementedException;
 import org.apache.hadoop.hbase.Cell;
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.regionserver.RegionScanner;
-import org.apache.kylin.common.util.FIFOIterable;
-import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.apache.kylin.common.util.Pair;
-import org.apache.kylin.common.util.StreamingBatch;
-import org.apache.kylin.common.util.StreamingMessage;
+import org.apache.kylin.common.util.*;
 import org.apache.kylin.invertedindex.IIInstance;
 import org.apache.kylin.invertedindex.IIManager;
 import org.apache.kylin.invertedindex.IISegment;
@@ -114,7 +110,7 @@ public class IITest extends LocalFileMetadataTestCase {
 @Nullable
 @Override
 public StreamingMessage apply(@Nullable MessageAndOffset input) {
-return parser.parse(input);
+return parser.parse(input.message().payload());
 }
 });
 StreamingBatch batch = new StreamingBatch(streamingMessages, 
Pair.newPair(0L, System.currentTimeMillis()));

http://git-wip-us.apache.org/repos/asf/kylin/blob/3202cb25/core-common/src/main/java/org/apache/kylin/common/util/StreamingMessage.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/util/StreamingMessage.java 
b/core-common/src/main/java/org/apache/kylin/common/util/StreamingMessage.java
index f327db2..53ab195 100644
--- 
a/core-common/src/main/java/org/apache/kylin/common/util/StreamingMessage.java
+++ 
b/core-common/src/main/java/org/apache/kylin/common/util/StreamingMessage.java
@@ -51,6 +51,10 @@ public class StreamingMessage {
 return offset;
 }
 
+public void 

[1/2] kylin git commit: refactor: more checks on finding segment by id

[mahongbin]

Repository: kylin
Updated Branches:
  refs/heads/master 9b8a9899b -> 91d6798d4


refactor: more checks on finding segment by id


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c736ee5f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c736ee5f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c736ee5f

Branch: refs/heads/master
Commit: c736ee5fe8009c28f5e016d24c5f113940bc11cf
Parents: 9b8a989
Author: Hongbin Ma 
Authored: Wed Jun 22 19:06:17 2016 +0800
Committer: Hongbin Ma 
Committed: Wed Jun 22 19:06:41 2016 +0800

--
 .../engine/mr/steps/SaveStatisticsStep.java | 48 ++--
 1 file changed, 43 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/c736ee5f/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
index 435fb13..7cc9dc3 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
@@ -21,6 +21,9 @@ package org.apache.kylin.engine.mr.steps;
 import java.io.IOException;
 import java.util.Random;
 
+import javax.annotation.Nullable;
+
+import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -43,9 +46,11 @@ import org.apache.kylin.metadata.model.MeasureDesc;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.google.common.base.Function;
+import com.google.common.collect.Iterables;
+
 /**
  * Save the cube segment statistic to Kylin metadata store
- *
  */
 public class SaveStatisticsStep extends AbstractExecutable {
 
@@ -55,12 +60,45 @@ public class SaveStatisticsStep extends AbstractExecutable {
 super();
 }
 
+private CubeSegment findSegment(ExecutableContext context, String 
cubeName, String segmentId) {
+final CubeManager mgr = CubeManager.getInstance(context.getConfig());
+final CubeInstance cube = mgr.getCube(cubeName);
+
+if (cube == null) {
+String cubeList = 
StringUtils.join(Iterables.transform(mgr.listAllCubes(), new 
Function() {
+@Nullable
+@Override
+public String apply(@Nullable CubeInstance input) {
+return input.getName();
+}
+}).iterator(), ",");
+
+logger.info("target cube name: {}, cube list: {}", cubeName, 
cubeList);
+throw new IllegalStateException();
+}
+
+
+final CubeSegment newSegment = cube.getSegmentById(segmentId);
+
+if (newSegment == null) {
+String segmentList = 
StringUtils.join(Iterables.transform(cube.getSegments(), new 
Function() {
+@Nullable
+@Override
+public String apply(@Nullable CubeSegment input) {
+return input.getUuid();
+}
+}).iterator(), ",");
+
+logger.info("target segment id: {}, segment list: {}", segmentId, 
segmentList);
+throw new IllegalStateException();
+}
+return newSegment;
+}
+
 @Override
 protected ExecuteResult doWork(ExecutableContext context) throws 
ExecuteException {
-final CubeManager mgr = CubeManager.getInstance(context.getConfig());
-final CubeInstance cube = 
mgr.getCube(CubingExecutableUtil.getCubeName(this.getParams()));
-final CubeSegment newSegment = 
cube.getSegmentById(CubingExecutableUtil.getSegmentId(this.getParams()));
-KylinConfig kylinConf = cube.getConfig();
+CubeSegment newSegment = findSegment(context, 
CubingExecutableUtil.getCubeName(this.getParams()), 
CubingExecutableUtil.getSegmentId(this.getParams()));
+KylinConfig kylinConf = newSegment.getConfig();
 
 ResourceStore rs = ResourceStore.getStore(kylinConf);
 try {

kylin git commit: refactor: subclass of AbstractExecutable change to use own logger

[mahongbin]

Repository: kylin
Updated Branches:
  refs/heads/master bfee5ed44 -> 94567c150


refactor: subclass of AbstractExecutable change to use own logger


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/94567c15
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/94567c15
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/94567c15

Branch: refs/heads/master
Commit: 94567c150e94d0f6499bb26621869c83d117c5a4
Parents: bfee5ed
Author: Hongbin Ma 
Authored: Wed Jun 22 17:51:55 2016 +0800
Committer: Hongbin Ma 
Committed: Wed Jun 22 17:52:03 2016 +0800

--
 .../kylin/job/common/ShellExecutable.java   |  4 
 .../kylin/job/execution/AbstractExecutable.java |  2 +-
 .../org/apache/kylin/engine/mr/CubingJob.java   |  4 
 .../engine/mr/common/HadoopShellExecutable.java |  4 
 .../UpdateIIInfoAfterBuildStep.java |  4 
 .../engine/mr/steps/MergeDictionaryStep.java|  5 
 .../engine/mr/steps/MergeStatisticsStep.java|  4 
 .../engine/mr/steps/SaveStatisticsStep.java |  4 
 .../mr/steps/UpdateCubeInfoAfterBuildStep.java  |  4 
 .../mr/steps/UpdateCubeInfoAfterMergeStep.java  |  4 
 .../kylin/engine/spark/SparkExecutable.java |  4 
 .../source/hive/CreateFlatHiveTableStep.java|  4 
 .../apache/kylin/source/hive/HiveMRInput.java   | 25 +++-
 .../apache/kylin/source/hive/HqlExecutable.java |  4 
 .../steps/HDFSPathGarbageCollectionStep.java|  5 
 15 files changed, 69 insertions(+), 12 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/94567c15/core-job/src/main/java/org/apache/kylin/job/common/ShellExecutable.java
--
diff --git 
a/core-job/src/main/java/org/apache/kylin/job/common/ShellExecutable.java 
b/core-job/src/main/java/org/apache/kylin/job/common/ShellExecutable.java
index 74f54a1..209d14e 100644
--- a/core-job/src/main/java/org/apache/kylin/job/common/ShellExecutable.java
+++ b/core-job/src/main/java/org/apache/kylin/job/common/ShellExecutable.java
@@ -32,11 +32,15 @@ import org.apache.kylin.job.execution.ExecutableContext;
 import org.apache.kylin.job.execution.ExecuteResult;
 
 import com.google.common.collect.Maps;
+import org.slf4j.LoggerFactory;
 
 /**
  */
 public class ShellExecutable extends AbstractExecutable {
 
+private static final org.slf4j.Logger logger = 
LoggerFactory.getLogger(ShellExecutable.class);
+
+
 private static final String CMD = "cmd";
 
 public ShellExecutable() {

http://git-wip-us.apache.org/repos/asf/kylin/blob/94567c15/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
--
diff --git 
a/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java 
b/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
index 395676b..778763c 100644
--- 
a/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
+++ 
b/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
@@ -49,7 +49,7 @@ public abstract class AbstractExecutable implements 
Executable, Idempotent {
 protected static final String START_TIME = "startTime";
 protected static final String END_TIME = "endTime";
 
-protected static final Logger logger = 
LoggerFactory.getLogger(AbstractExecutable.class);
+private static final Logger logger = 
LoggerFactory.getLogger(AbstractExecutable.class);
 protected int retry = 0;
 
 private String name;

http://git-wip-us.apache.org/repos/asf/kylin/blob/94567c15/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java
--
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java
index 70b6132..bbd0fc5 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java
@@ -42,11 +42,15 @@ import org.apache.kylin.job.execution.ExecuteResult;
 import org.apache.kylin.job.execution.Output;
 import org.apache.kylin.metadata.project.ProjectInstance;
 import org.apache.kylin.metadata.project.ProjectManager;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  */
 public class CubingJob extends DefaultChainedExecutable {
 
+private static final Logger logger = 
LoggerFactory.getLogger(CubingJob.class);
+
 public enum AlgorithmEnum {
 LAYER, INMEM
 }

[1/3] kylin git commit: refactor: minor changes

[mahongbin]

Repository: kylin
Updated Branches:
  refs/heads/master ff1c81395 -> b361d8aaa


refactor: minor changes


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b361d8aa
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b361d8aa
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b361d8aa

Branch: refs/heads/master
Commit: b361d8aaa2b1b5d0d86107bedb9f34397be000a5
Parents: ea24450
Author: Hongbin Ma 
Authored: Wed Jun 22 13:59:58 2016 +0800
Committer: Hongbin Ma 
Committed: Wed Jun 22 14:00:15 2016 +0800

--
 .../org/apache/kylin/cube/kv/RowKeyEncoder.java | 21 ++-
 .../org/apache/kylin/gridtable/GTRecord.java| 17 +-
 .../gridtable/memstore/GTSimpleMemStore.java| 20 +-
 .../kylin/gridtable/DictGridTableTest.java  | 64 ++--
 .../coprocessor/endpoint/CubeVisitService.java  |  2 +-
 5 files changed, 70 insertions(+), 54 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/b361d8aa/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoder.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoder.java 
b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoder.java
index db704dc..ebcbadd 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoder.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoder.java
@@ -94,12 +94,31 @@ public class RowKeyEncoder extends AbstractRowKeyEncoder {
 @Override
 public void encode(GTRecord record, ImmutableBitSet keyColumns, byte[] 
buf) {
 ByteArray byteArray = new ByteArray(buf, getHeaderLength(), 0);
-record.exportColumns(keyColumns, byteArray, defaultValue());
+
+encodeDims(record, keyColumns, byteArray, defaultValue());
 
 //fill shard and cuboid
 fillHeader(buf);
 }
 
+//ByteArray representing dimension does not have extra header
+public void encodeDims(GTRecord record, ImmutableBitSet selectedCols, 
ByteArray buf, byte defaultValue) {
+int pos = 0;
+for (int i = 0; i < selectedCols.trueBitCount(); i++) {
+int c = selectedCols.trueBitAt(i);
+ByteArray columnC = record.get(c);
+if (columnC.array() != null) {
+System.arraycopy(record.get(c).array(), columnC.offset(), 
buf.array(), buf.offset() + pos, columnC.length());
+pos += columnC.length();
+} else {
+int maxLength = 
record.getInfo().getCodeSystem().maxCodeLength(c);
+Arrays.fill(buf.array(), buf.offset() + pos, buf.offset() + 
pos + maxLength, defaultValue);
+pos += maxLength;
+}
+}
+buf.setLength(pos);
+}
+
 @Override
 public void encode(ByteArray bodyBytes, ByteArray outputBuf) {
 Preconditions.checkState(bodyBytes.length() == bodyLength);

http://git-wip-us.apache.org/repos/asf/kylin/blob/b361d8aa/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java 
b/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java
index b676693..276f66e 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java
@@ -235,22 +235,7 @@ public class GTRecord implements Comparable {
 buf.setLength(pos);
 }
 
-/** write data to given buffer, like serialize, use defaultValue when 
required column is not set*/
-public void exportColumns(ImmutableBitSet selectedCols, ByteArray buf, 
byte defaultValue) {
-int pos = 0;
-for (int i = 0; i < selectedCols.trueBitCount(); i++) {
-int c = selectedCols.trueBitAt(i);
-if (cols[c].array() != null) {
-System.arraycopy(cols[c].array(), cols[c].offset(), 
buf.array(), buf.offset() + pos, cols[c].length());
-pos += cols[c].length();
-} else {
-int maxLength = info.codeSystem.maxCodeLength(c);
-Arrays.fill(buf.array(), buf.offset() + pos, buf.offset() + 
pos + maxLength, defaultValue);
-pos += maxLength;
-}
-}
-buf.setLength(pos);
-}
+   
 
 /** write data to given buffer, like serialize */
 public void exportColumns(ImmutableBitSet selectedCols, ByteBuffer buf) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/b361d8aa/core-cube/src/main/java/org/apache/kylin/gridtable/memstore/GTSimpleMemStore.java
--
diff 

[12/28] kylin git commit: refactor

[lidong]

refactor


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/bdffa9fb
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/bdffa9fb
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/bdffa9fb

Branch: refs/heads/1.5.x-HBase1.x
Commit: bdffa9fbe46b23765e2d71fa0a2ddf7ff55bd1d8
Parents: 0870a6a
Author: Hongbin Ma 
Authored: Mon May 9 18:52:23 2016 +0800
Committer: Hongbin Ma 
Committed: Mon May 9 18:52:23 2016 +0800

--
 .../src/main/java/org/apache/kylin/common/KylinConfigBase.java   | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/bdffa9fb/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 3db97b3..2e17c94 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -366,8 +366,8 @@ abstract public class KylinConfigBase implements 
Serializable {
 return Integer.parseInt(getOptional("kylin.rest.workers.per.server", 
"1"));
 }
 
-public String getAdminDls() {
-return getOptional("kylin.job.admin.dls", null);
+public String[] getAdminDls() {
+return getOptionalStringArray("kylin.job.admin.dls", null);
 }
 
 public long getJobStepTimeout() {

kylin git commit: refactor

[mahongbin]

Repository: kylin
Updated Branches:
  refs/heads/master 656c645f3 -> 2fc706ca1


refactor


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2fc706ca
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2fc706ca
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2fc706ca

Branch: refs/heads/master
Commit: 2fc706ca132dfad43b2f68c41757ddfbadc595b0
Parents: 656c645
Author: Hongbin Ma 
Authored: Mon May 9 18:38:13 2016 +0800
Committer: Hongbin Ma 
Committed: Mon May 9 18:40:06 2016 +0800

--
 build/conf/kylin.properties |  3 +-
 .../apache/kylin/common/KylinConfigBase.java| 14 +++-
 .../kylin/common/debug/BackdoorToggles.java | 24 ++
 .../kylin/common/restclient/RestClient.java | 16 ++--
 .../common/util/LoggableCachedThreadPool.java   | 57 ++
 .../kylin/common/util/NumberIterators.java  | 34 
 .../apache/kylin/gridtable/ScannerWorker.java   |  3 +-
 .../apache/kylin/storage/cache/EhcacheTest.java | 49 
 .../apache/kylin/rest/response/SQLResponse.java | 26 +++
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 82 +++-
 10 files changed, 213 insertions(+), 95 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/2fc706ca/build/conf/kylin.properties
--
diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties
index 80ae004..9a2fd81 100644
--- a/build/conf/kylin.properties
+++ b/build/conf/kylin.properties
@@ -168,4 +168,5 @@ kylin.web.hive.limit=20
 # E.g, for cube whose capacity be marked as "SMALL", split region per 5GB by 
default
 kylin.hbase.region.cut.small=5
 kylin.hbase.region.cut.medium=10
-kylin.hbase.region.cut.large=50
\ No newline at end of file
+kylin.hbase.region.cut.large=50
+

http://git-wip-us.apache.org/repos/asf/kylin/blob/2fc706ca/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 850ea17..3db97b3 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -361,8 +361,13 @@ abstract public class KylinConfigBase implements 
Serializable {
 return getOptionalStringArray("kylin.rest.servers", new String[0]);
 }
 
-public String[] getAdminDls() {
-return getOptionalStringArray("kylin.job.admin.dls", null);
+public int getWorkersPerServer() {
+//for sequence sql use
+return Integer.parseInt(getOptional("kylin.rest.workers.per.server", 
"1"));
+}
+
+public String getAdminDls() {
+return getOptional("kylin.job.admin.dls", null);
 }
 
 public long getJobStepTimeout() {
@@ -498,6 +503,10 @@ abstract public class KylinConfigBase implements 
Serializable {
 return 
Integer.parseInt(this.getOptional("kylin.hbase.client.keyvalue.maxsize", 
"10485760"));
 }
 
+public String getDefaultIGTStorage() {
+return getOptional("kylin.query.storage.default.gtstorage", 
"org.apache.kylin.storage.hbase.cube.v2.CubeHBaseEndpointRPC");
+}
+
 public int getHBaseScanCacheRows() {
 return 
Integer.parseInt(this.getOptional("kylin.hbase.scan.cache_rows", "1024"));
 }
@@ -632,4 +641,5 @@ abstract public class KylinConfigBase implements 
Serializable {
 public int getCubeStatsHLLPrecision() {
 return 
Integer.parseInt(getOptional("kylin.job.cubing.inmem.sampling.hll.precision", 
"14"));
 }
+
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/2fc706ca/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java 
b/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
index 0feff2d..f88781a 100644
--- 
a/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
+++ 
b/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
@@ -20,6 +20,9 @@ package org.apache.kylin.common.debug;
 
 import java.util.Map;
 
+import org.apache.commons.lang.StringUtils;
+import org.apache.kylin.common.util.Pair;
+
 /**
  */
 public class BackdoorToggles {
@@ -62,6 +65,16 @@ public class BackdoorToggles {
 return Integer.valueOf(v);
 }
 
+public static Pair getShardAssignment() {
+String v = getString(DEBUG_TOGGLE_SHARD_ASSIGNMENT);
+if (v == 

[5/8] kylin git commit: refactor

[mahongbin]

refactor


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b26b2489
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b26b2489
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b26b2489

Branch: refs/heads/master
Commit: b26b2489baddcfd148eaf8f17330878bbf349048
Parents: aaf3b87
Author: Hongbin Ma 
Authored: Mon Apr 11 17:33:07 2016 +0800
Committer: Hongbin Ma 
Committed: Wed Apr 13 11:11:15 2016 +0800

--
 .../kylin/job/DeployLocalMetaToRemoteTest.java  |   4 +-
 .../java/org/apache/kylin/job/DeployUtil.java   |   4 +-
 .../org/apache/kylin/common/util/ByteArray.java |  11 +-
 .../kylin/common/util/Log4jConfigurer.java  |   2 +
 .../common/util/AbstractKylinTestCase.java  |  14 +-
 .../common/util/HBaseMetadataTestCase.java  |   4 +-
 .../common/util/LocalFileMetadataTestCase.java  |   8 +-
 .../org/apache/kylin/cube/cuboid/Cuboid.java|  18 ++
 .../apache/kylin/cube/kv/LazyRowKeyEncoder.java |  25 --
 .../org/apache/kylin/cube/model/CubeDesc.java   |   5 +
 .../org/apache/kylin/gridtable/GTRecord.java|  10 +-
 .../org/apache/kylin/gridtable/GTScanRange.java |   2 -
 .../kylin/gridtable/GTScanRangePlanner.java | 294 ---
 .../apache/kylin/gridtable/GTScanRequest.java   |  68 +++--
 .../org/apache/kylin/gridtable/IGTStorage.java  |  27 ++
 .../apache/kylin/gridtable/ScannerWorker.java   |  72 +
 .../kylin/cube/AggregationGroupRuleTest.java|  19 +-
 .../apache/kylin/cube/RowKeyAttrRuleTest.java   |   5 +-
 .../cube/inmemcubing/DoggedCubeBuilderTest.java | 160 --
 .../cube/inmemcubing/InMemCubeBuilderTest.java  | 268 -
 .../kylin/gridtable/DictGridTableTest.java  |  58 ++--
 .../impl/threadpool/DefaultSchedulerTest.java   | 151 --
 .../filter/UDF/MassInValueProviderFactory.java  |   1 -
 .../apache/kylin/storage/StorageMockUtils.java  | 189 
 .../kylin/storage/cache/StorageMockUtils.java   | 157 --
 kylin-it/pom.xml|   7 +
 .../ITDoggedCubeBuilderStressTest.java  |   8 +-
 .../inmemcubing/ITDoggedCubeBuilderTest.java| 163 ++
 .../inmemcubing/ITInMemCubeBuilderTest.java | 271 +
 .../impl/threadpool/ITDefaultSchedulerTest.java | 154 ++
 .../kylin/provision/BuildCubeWithEngine.java|   4 +-
 .../kylin/provision/BuildCubeWithSpark.java |   6 +-
 .../kylin/provision/BuildCubeWithStream.java|   4 +-
 .../kylin/provision/BuildIIWithEngine.java  |   4 +-
 .../kylin/provision/BuildIIWithStream.java  |   2 +-
 .../org/apache/kylin/query/KylinTestBase.java   |  11 -
 .../kylin/storage/hbase/ITStorageTest.java  |   2 +-
 source-hive/pom.xml |   7 +
 .../kylin/source/hive/HiveCmdBuilderTest.java   |   6 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 111 ---
 .../storage/hbase/cube/v2/CubeHBaseRPC.java |  56 +---
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  94 --
 .../hbase/cube/v2/CubeSegmentScanner.java   | 215 +-
 .../storage/hbase/cube/v2/CubeStorageQuery.java |  30 +-
 .../kylin/storage/hbase/cube/v2/RawScan.java|  10 +
 .../coprocessor/endpoint/CubeVisitService.java  |  88 --
 .../hbase/steps/SandboxMetastoreCLI.java|   2 +-
 47 files changed, 1539 insertions(+), 1292 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/b26b2489/assembly/src/test/java/org/apache/kylin/job/DeployLocalMetaToRemoteTest.java
--
diff --git 
a/assembly/src/test/java/org/apache/kylin/job/DeployLocalMetaToRemoteTest.java 
b/assembly/src/test/java/org/apache/kylin/job/DeployLocalMetaToRemoteTest.java
index 1267ab7..8494607 100644
--- 
a/assembly/src/test/java/org/apache/kylin/job/DeployLocalMetaToRemoteTest.java
+++ 
b/assembly/src/test/java/org/apache/kylin/job/DeployLocalMetaToRemoteTest.java
@@ -45,7 +45,7 @@ public class DeployLocalMetaToRemoteTest {
 public static void beforeClass() throws Exception {
 logger.info("Adding to classpath: " + new 
File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
 ClassUtil.addClasspath(new 
File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
-System.setProperty(KylinConfig.KYLIN_CONF, 
"../examples/test_case_data/sandbox");
+System.setProperty(KylinConfig.KYLIN_CONF, 
HBaseMetadataTestCase.SANDBOX_TEST_DATA);
 if (StringUtils.isEmpty(System.getProperty("hdp.version"))) {
 throw new RuntimeException("No hdp.version set; Please set 
hdp.version in your jvm option, for example: -Dhdp.version=2.2.4.2-2");
 }
@@ -53,7 +53,7 @@ public class DeployLocalMetaToRemoteTest {
 
 @Before
 

[3/8] kylin git commit: refactor

[mahongbin]

http://git-wip-us.apache.org/repos/asf/kylin/blob/b26b2489/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
index ef53cb7..938145b 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
@@ -19,30 +19,35 @@
 package org.apache.kylin.storage.hbase.cube.v2;
 
 import java.io.IOException;
+import java.util.Arrays;
 import java.util.Iterator;
 import java.util.List;
 
-import javax.annotation.Nullable;
-
 import org.apache.hadoop.hbase.Cell;
 import org.apache.hadoop.hbase.client.HConnection;
 import org.apache.hadoop.hbase.client.HTableInterface;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.ResultScanner;
 import org.apache.hadoop.hbase.client.Scan;
+import org.apache.kylin.common.util.BytesUtil;
 import org.apache.kylin.common.util.ImmutableBitSet;
+import org.apache.kylin.common.util.ShardingHash;
 import org.apache.kylin.cube.CubeSegment;
 import org.apache.kylin.cube.cuboid.Cuboid;
+import org.apache.kylin.cube.kv.RowConstants;
+import org.apache.kylin.dimension.DimensionEncoding;
 import org.apache.kylin.gridtable.GTInfo;
 import org.apache.kylin.gridtable.GTRecord;
 import org.apache.kylin.gridtable.GTScanRequest;
 import org.apache.kylin.gridtable.IGTScanner;
 import org.apache.kylin.gridtable.IGTStore;
+import org.apache.kylin.metadata.filter.UDF.MassInTupleFilter;
+import org.apache.kylin.metadata.model.TblColRef;
 import org.apache.kylin.storage.hbase.HBaseConnection;
+import 
org.apache.kylin.storage.hbase.cube.v2.filter.MassInValueProviderFactoryImpl;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.google.common.base.Function;
 import com.google.common.collect.Iterators;
 import com.google.common.collect.Lists;
 
@@ -84,53 +89,77 @@ public class CubeHBaseScanRPC extends CubeHBaseRPC {
 }
 }
 
-public CubeHBaseScanRPC(CubeSegment cubeSeg, Cuboid cuboid, GTInfo 
fullGTInfo) {
+public CubeHBaseScanRPC(CubeSegment cubeSeg, Cuboid cuboid, final GTInfo 
fullGTInfo) {
 super(cubeSeg, cuboid, fullGTInfo);
+MassInTupleFilter.VALUE_PROVIDER_FACTORY = new 
MassInValueProviderFactoryImpl(new MassInValueProviderFactoryImpl.DimEncAware() 
{
+@Override
+public DimensionEncoding getDimEnc(TblColRef col) {
+return 
fullGTInfo.getCodeSystem().getDimEnc(col.getColumnDesc().getZeroBasedIndex());
+}
+});
 }
 
 @Override
-public IGTScanner getGTScanner(final List scanRequests) 
throws IOException {
-final List scanners = Lists.newArrayList();
-for (GTScanRequest request : scanRequests) {
-scanners.add(getGTScanner(request));
-}
+public IGTScanner getGTScanner(final GTScanRequest scanRequest) throws 
IOException {
+final IGTScanner scanner = getGTScannerInternal(scanRequest);
 
 return new IGTScanner() {
 @Override
 public GTInfo getInfo() {
-return scanners.get(0).getInfo();
+return scanner.getInfo();
 }
 
 @Override
 public int getScannedRowCount() {
 int sum = 0;
-for (IGTScanner s : scanners) {
-sum += s.getScannedRowCount();
-}
+sum += scanner.getScannedRowCount();
 return sum;
 }
 
 @Override
 public void close() throws IOException {
-for (IGTScanner s : scanners) {
-s.close();
-}
+scanner.close();
 }
 
 @Override
 public Iterator iterator() {
-return 
Iterators.concat(Iterators.transform(scanners.iterator(), new 
Function() {
-@Nullable
-@Override
-public Iterator apply(IGTScanner input) {
-return input.iterator();
-}
-}));
+return scanner.iterator();
 }
 };
 }
 
-private IGTScanner getGTScanner(final GTScanRequest scanRequest) throws 
IOException {
+//for non-sharding cases it will only return one byte[] with not shard at 
beginning
+private List getRowKeysDifferentShards(byte[] halfCookedKey) {
+final short cuboidShardNum = cubeSeg.getCuboidShardNum(cuboid.getId());
+
+if (!cubeSeg.isEnableSharding()) {
+return Lists.newArrayList(halfCookedKey);//not shard to append at 
head, 

[4/8] kylin git commit: refactor

[mahongbin]

http://git-wip-us.apache.org/repos/asf/kylin/blob/b26b2489/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/DefaultSchedulerTest.java
--
diff --git 
a/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/DefaultSchedulerTest.java
 
b/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/DefaultSchedulerTest.java
deleted file mode 100644
index d50baad..000
--- 
a/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/DefaultSchedulerTest.java
+++ /dev/null
@@ -1,151 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- * http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.job.impl.threadpool;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.ScheduledFuture;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.kylin.job.BaseTestExecutable;
-import org.apache.kylin.job.ErrorTestExecutable;
-import org.apache.kylin.job.FailedTestExecutable;
-import org.apache.kylin.job.SelfStopExecutable;
-import org.apache.kylin.job.SucceedTestExecutable;
-import org.apache.kylin.job.execution.DefaultChainedExecutable;
-import org.apache.kylin.job.execution.ExecutableState;
-import org.junit.Test;
-
-/**
- */
-public class DefaultSchedulerTest extends BaseSchedulerTest {
-
-@Test
-public void testSingleTaskJob() throws Exception {
-DefaultChainedExecutable job = new DefaultChainedExecutable();
-BaseTestExecutable task1 = new SucceedTestExecutable();
-job.addTask(task1);
-jobService.addJob(job);
-waitForJobFinish(job.getId());
-assertEquals(ExecutableState.SUCCEED, 
jobService.getOutput(job.getId()).getState());
-assertEquals(ExecutableState.SUCCEED, 
jobService.getOutput(task1.getId()).getState());
-}
-
-@Test
-public void testSucceed() throws Exception {
-DefaultChainedExecutable job = new DefaultChainedExecutable();
-BaseTestExecutable task1 = new SucceedTestExecutable();
-BaseTestExecutable task2 = new SucceedTestExecutable();
-job.addTask(task1);
-job.addTask(task2);
-jobService.addJob(job);
-waitForJobFinish(job.getId());
-assertEquals(ExecutableState.SUCCEED, 
jobService.getOutput(job.getId()).getState());
-assertEquals(ExecutableState.SUCCEED, 
jobService.getOutput(task1.getId()).getState());
-assertEquals(ExecutableState.SUCCEED, 
jobService.getOutput(task2.getId()).getState());
-}
-
-@Test
-public void testSucceedAndFailed() throws Exception {
-DefaultChainedExecutable job = new DefaultChainedExecutable();
-BaseTestExecutable task1 = new SucceedTestExecutable();
-BaseTestExecutable task2 = new FailedTestExecutable();
-job.addTask(task1);
-job.addTask(task2);
-jobService.addJob(job);
-waitForJobFinish(job.getId());
-assertEquals(ExecutableState.ERROR, 
jobService.getOutput(job.getId()).getState());
-assertEquals(ExecutableState.SUCCEED, 
jobService.getOutput(task1.getId()).getState());
-assertEquals(ExecutableState.ERROR, 
jobService.getOutput(task2.getId()).getState());
-}
-
-@Test
-public void testSucceedAndError() throws Exception {
-DefaultChainedExecutable job = new DefaultChainedExecutable();
-BaseTestExecutable task1 = new ErrorTestExecutable();
-BaseTestExecutable task2 = new SucceedTestExecutable();
-job.addTask(task1);
-job.addTask(task2);
-jobService.addJob(job);
-waitForJobFinish(job.getId());
-assertEquals(ExecutableState.ERROR, 
jobService.getOutput(job.getId()).getState());
-assertEquals(ExecutableState.ERROR, 
jobService.getOutput(task1.getId()).getState());
-assertEquals(ExecutableState.READY, 
jobService.getOutput(task2.getId()).getState());
-}
-
-@Test
-public void testDiscard() throws Exception {
-DefaultChainedExecutable job = new