kylin git commit: remove unnecessary raw measure cond

2016-09-09 Thread mahongbin 
Repository: kylin
Updated Branches:
  refs/heads/master 942406bda -> a05f11146


remove unnecessary raw measure cond


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a05f1114
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a05f1114
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a05f1114

Branch: refs/heads/master
Commit: a05f11146c9edd4ba18618f38fd13c9b7e4806ce
Parents: 942406b
Author: Hongbin Ma 
Authored: Fri Sep 9 23:45:17 2016 +0800
Committer: Hongbin Ma 
Committed: Fri Sep 9 23:45:17 2016 +0800

--
 .../localmeta/cube_desc/test_kylin_cube_without_slr_desc.json | 3 ---
 .../cube_desc/test_kylin_cube_without_slr_left_join_desc.json | 3 ---
 2 files changed, 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/a05f1114/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json
--
diff --git 
a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json
 
b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json
index 7de2ae2..d185175 100644
--- 
a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json
+++ 
b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json
@@ -234,9 +234,6 @@
   "item_count_sum",
   "SITE_EXTENDED_1",
   "SITE_EXTENDED_2",
-  "CAL_DT_RAW",
-  "LSTG_FORMAT_NAME_RAW",
-  "LEAF_CATEG_ID_RAW",
   "PRICE_RAW"
 ]
   }

http://git-wip-us.apache.org/repos/asf/kylin/blob/a05f1114/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
--
diff --git 
a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
 
b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
index 4270aab..2aea1a8 100644
--- 
a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
+++ 
b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
@@ -266,9 +266,6 @@
   "gmv_max",
   "trans_cnt",
   "item_count_sum",
-  "CAL_DT_RAW",
-  "LSTG_FORMAT_NAME_RAW",
-  "LEAF_CATEG_ID_RAW",
   "PRICE_RAW"
 ]
   }

kylin git commit: KYLIN-1922 imporve CI

2016-09-09 Thread mahongbin 
Repository: kylin
Updated Branches:
  refs/heads/master 466cf1afb -> 942406bda


KYLIN-1922 imporve CI


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/942406bd
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/942406bd
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/942406bd

Branch: refs/heads/master
Commit: 942406bda3ec7405a6d2be27ba11bb38b5f88298
Parents: 466cf1a
Author: Hongbin Ma 
Authored: Fri Sep 9 23:01:00 2016 +0800
Committer: Hongbin Ma 
Committed: Fri Sep 9 23:01:00 2016 +0800

--
 .../apache/kylin/query/ITKylinQueryTest.java| 22 
 1 file changed, 18 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/942406bd/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java 
b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
index 5f6af7a..3411c91 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
@@ -44,6 +44,8 @@ import org.apache.kylin.storage.hbase.HBaseStorage;
 import 
org.apache.kylin.storage.hbase.cube.v1.coprocessor.observer.ObserverEnabler;
 import org.dbunit.database.DatabaseConnection;
 import org.dbunit.database.IDatabaseConnection;
+import org.hamcrest.BaseMatcher;
+import org.hamcrest.Description;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.Ignore;
@@ -119,6 +121,22 @@ public class ITKylinQueryTest extends KylinTestBase {
 
 @Test
 public void testTimeoutQuery() throws Exception {
+thrown.expect(SQLException.class);
+
+//should not break at table duplicate check, should fail at model 
duplicate check
+thrown.expectCause(new BaseMatcher() {
+@Override
+public boolean matches(Object item) {
+if (item instanceof GTScanSelfTerminatedException) {
+return true;
+}
+return false;
+}
+
+@Override
+public void describeTo(Description description) {
+}
+});
 
 try {
 
@@ -133,10 +151,6 @@ public class ITKylinQueryTest extends KylinTestBase {
 
RemoveBlackoutRealizationsRule.blackouts.add("CUBE[name=test_kylin_cube_without_slr_inner_join_empty]");
 
 execAndCompQuery(getQueryFolderPrefix() + 
"src/test/resources/query/sql_timeout", null, true);
-} catch (SQLException e) {
-if (!(e.getCause() instanceof GTScanSelfTerminatedException)) {
-throw new RuntimeException();
-}
 } finally {
 
 //these two cubes has RAW measure, will disturb limit push down

[1/2] kylin git commit: minor, remove unnecessary raw measures

2016-09-09 Thread mahongbin 
Repository: kylin
Updated Branches:
  refs/heads/master a2c875d8a -> 466cf1afb


minor, remove unnecessary raw measures


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/618cf28c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/618cf28c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/618cf28c

Branch: refs/heads/master
Commit: 618cf28c96d6b267d13ff737d2b3c550fc67e176
Parents: a2c875d
Author: Hongbin Ma 
Authored: Fri Sep 9 18:57:01 2016 +0800
Committer: Hongbin Ma 
Committed: Fri Sep 9 18:57:01 2016 +0800

--
 .../kylin/measure/topn/TopNMeasureType.java |   2 +
 .../test_case_data/localmeta/cube_desc/ssb.json | 409 +++--
 .../test_kylin_cube_with_slr_desc.json  | 389 +++-
 ...st_kylin_cube_with_view_inner_join_desc.json | 388 +++-
 ...est_kylin_cube_with_view_left_join_desc.json | 388 +++-
 .../test_kylin_cube_without_slr_desc.json   |  58 +-
 ...t_kylin_cube_without_slr_left_join_desc.json | 587 +++
 .../test_streaming_table_cube_desc.json | 245 
 8 files changed, 1423 insertions(+), 1043 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/618cf28c/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
 
b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
index ed22d61..0756056 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
@@ -274,6 +274,8 @@ public class TopNMeasureType extends 
MeasureType {
 
 if (sum.isSum() == false)
 return false;
+if (sum.getParameter() == null || sum.getParameter().getColRefs() == 
null || sum.getParameter().getColRefs().size() == 0)
+return false;
 
 TblColRef sumCol = sum.getParameter().getColRefs().get(0);
 return sumCol.equals(topnNumCol);

http://git-wip-us.apache.org/repos/asf/kylin/blob/618cf28c/examples/test_case_data/localmeta/cube_desc/ssb.json
--
diff --git a/examples/test_case_data/localmeta/cube_desc/ssb.json 
b/examples/test_case_data/localmeta/cube_desc/ssb.json
index d3ea10b..4903979 100644
--- a/examples/test_case_data/localmeta/cube_desc/ssb.json
+++ b/examples/test_case_data/localmeta/cube_desc/ssb.json
@@ -1,179 +1,256 @@
 {
-  "uuid" : "5c44df30-daec-486e-af90-927bf7851057",
-  "name" : "ssb",
-  "description" : "",
-  "dimensions" : [ {
-"name" : "SSB.PART_DERIVED",
-"table" : "SSB.PART",
-"column" : null,
-"derived" : [ "P_MFGR", "P_CATEGORY", "P_BRAND" ]
-  }, {
-"name" : "C_CITY",
-"table" : "SSB.CUSTOMER",
-"column" : "C_CITY",
-"derived" : null
-  }, {
-"name" : "C_REGION",
-"table" : "SSB.CUSTOMER",
-"column" : "C_REGION",
-"derived" : null
-  }, {
-"name" : "C_NATION",
-"table" : "SSB.CUSTOMER",
-"column" : "C_NATION",
-"derived" : null
-  }, {
-"name" : "S_CITY",
-"table" : "SSB.SUPPLIER",
-"column" : "S_CITY",
-"derived" : null
-  }, {
-"name" : "S_REGION",
-"table" : "SSB.SUPPLIER",
-"column" : "S_REGION",
-"derived" : null
-  }, {
-"name" : "S_NATION",
-"table" : "SSB.SUPPLIER",
-"column" : "S_NATION",
-"derived" : null
-  }, {
-"name" : "D_YEAR",
-"table" : "SSB.DATES",
-"column" : "D_YEAR",
-"derived" : null
-  }, {
-"name" : "D_YEARMONTH",
-"table" : "SSB.DATES",
-"column" : "D_YEARMONTH",
-"derived" : null
-  }, {
-"name" : "D_YEARMONTHNUM",
-"table" : "SSB.DATES",
-"column" : "D_YEARMONTHNUM",
-"derived" : null
-  }, {
-"name" : "D_WEEKNUMINYEAR",
-"table" : "SSB.DATES",
-"column" : "D_WEEKNUMINYEAR",
-"derived" : null
-  } ],
-  "measures" : [ {
-"name" : "_COUNT_",
-"function" : {
-  "expression" : "COUNT",
-  "parameter" : {
-"type" : "constant",
-"value" : "1",
-"next_parameter" : null
-  },
-  "returntype" : "bigint"
+  "uuid": "5c44df30-daec-486e-af90-927bf7851057",
+  "name": "ssb",
+  "description": "",
+  "dimensions": [
+{
+  "name": "SSB.PART_DERIVED",
+  "table": "SSB.PART",
+  "column": null,
+  "derived": [
+"P_MFGR",
+"P_CATEGORY",
+"P_BRAND"
+  ]
+},
+{
+  "name": "C_CITY",
+  "table": "SSB.CUSTOMER",
+  "column": "C_CITY",
+  "derived": null
+},
+{
+  "name": "C_REGION",
+  "table":

[2/2] kylin git commit: KYLIN-1922 fix CI

2016-09-09 Thread mahongbin 
KYLIN-1922 fix CI


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/466cf1af
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/466cf1af
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/466cf1af

Branch: refs/heads/master
Commit: 466cf1afb19ee1acb4f99ff72ac61e7689617957
Parents: 618cf28
Author: Hongbin Ma 
Authored: Fri Sep 9 22:27:17 2016 +0800
Committer: Hongbin Ma 
Committed: Fri Sep 9 22:27:17 2016 +0800

--
 .../apache/kylin/query/ITKylinQueryTest.java| 53 
 1 file changed, 21 insertions(+), 32 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/466cf1af/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java 
b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
index 0efea64..5f6af7a 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
@@ -32,6 +32,7 @@ import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.debug.BackdoorToggles;
 import org.apache.kylin.common.util.HBaseMetadataTestCase;
 import org.apache.kylin.gridtable.GTScanSelfTerminatedException;
+import org.apache.kylin.gridtable.StorageSideBehavior;
 import org.apache.kylin.metadata.project.ProjectInstance;
 import org.apache.kylin.metadata.realization.RealizationType;
 import org.apache.kylin.query.enumerator.OLAPQuery;
@@ -40,12 +41,9 @@ import org.apache.kylin.query.routing.Candidate;
 import org.apache.kylin.query.routing.rules.RemoveBlackoutRealizationsRule;
 import org.apache.kylin.query.schema.OLAPSchemaFactory;
 import org.apache.kylin.storage.hbase.HBaseStorage;
-import org.apache.kylin.gridtable.StorageSideBehavior;
 import 
org.apache.kylin.storage.hbase.cube.v1.coprocessor.observer.ObserverEnabler;
 import org.dbunit.database.DatabaseConnection;
 import org.dbunit.database.IDatabaseConnection;
-import org.hamcrest.BaseMatcher;
-import org.hamcrest.Description;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.Ignore;
@@ -122,41 +120,32 @@ public class ITKylinQueryTest extends KylinTestBase {
 @Test
 public void testTimeoutQuery() throws Exception {
 
-thrown.expect(SQLException.class);
-
-//should not break at table duplicate check, should fail at model 
duplicate check
-thrown.expectCause(new BaseMatcher() {
-@Override
-public boolean matches(Object item) {
-if (item instanceof GTScanSelfTerminatedException) {
-return true;
-}
-return false;
-}
-
-@Override
-public void describeTo(Description description) {
-}
-});
+try {
 
-Map toggles = Maps.newHashMap();
-toggles.put(BackdoorToggles.DEBUG_TOGGLE_COPROCESSOR_BEHAVIOR, 
StorageSideBehavior.SCAN_FILTER_AGGR_CHECKMEM_WITHDELAY.toString());//delay 
10ms for every scan
-BackdoorToggles.setToggles(toggles);
+Map toggles = Maps.newHashMap();
+toggles.put(BackdoorToggles.DEBUG_TOGGLE_COPROCESSOR_BEHAVIOR, 
StorageSideBehavior.SCAN_FILTER_AGGR_CHECKMEM_WITHDELAY.toString());//delay 
10ms for every scan
+BackdoorToggles.setToggles(toggles);
 
-
KylinConfig.getInstanceFromEnv().setProperty("kylin.query.cube.visit.timeout.times",
 "0.03");//set timeout to 9s
+
KylinConfig.getInstanceFromEnv().setProperty("kylin.query.cube.visit.timeout.times",
 "0.03");//set timeout to 9s
 
-//these two cubes has RAW measure, will disturb limit push down
-
RemoveBlackoutRealizationsRule.blackouts.add("CUBE[name=test_kylin_cube_without_slr_left_join_empty]");
-
RemoveBlackoutRealizationsRule.blackouts.add("CUBE[name=test_kylin_cube_without_slr_inner_join_empty]");
+//these two cubes has RAW measure, will disturb limit push down
+
RemoveBlackoutRealizationsRule.blackouts.add("CUBE[name=test_kylin_cube_without_slr_left_join_empty]");
+
RemoveBlackoutRealizationsRule.blackouts.add("CUBE[name=test_kylin_cube_without_slr_inner_join_empty]");
 
-execAndCompQuery(getQueryFolderPrefix() + 
"src/test/resources/query/sql_timeout", null, true);
+execAndCompQuery(getQueryFolderPrefix() + 
"src/test/resources/query/sql_timeout", null, true);
+} catch (SQLException e) {
+if (!(e.getCause() instanceof GTScanSelfTerminatedException)) {
+throw new RuntimeException();
+}
+

kylin git commit: KYLIN-2004 Make the creating intermediate hive table steps configurable

2016-09-09 Thread shaofengshi 
Repository: kylin
Updated Branches:
  refs/heads/1.5.4-rc1 639e4b015 -> 7c5c6c1c6


KYLIN-2004 Make the creating intermediate hive table steps configurable

Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7c5c6c1c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7c5c6c1c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7c5c6c1c

Branch: refs/heads/1.5.4-rc1
Commit: 7c5c6c1c681154d62adbf18285b80f3ebed7204b
Parents: 639e4b0
Author: shaofengshi 
Authored: Fri Sep 9 19:04:10 2016 +0800
Committer: shaofengshi 
Committed: Fri Sep 9 19:04:23 2016 +0800

--
 .../apache/kylin/common/KylinConfigBase.java|   4 +-
 .../org/apache/kylin/job/JoinedFlatTable.java   |  48 --
 .../kylin/job/constant/ExecutableConstants.java |   1 +
 .../kylin/job/execution/AbstractExecutable.java |   2 +-
 .../apache/kylin/job/JoinedFlatTableTest.java   |   2 +-
 .../kylin/metadata/model/DataModelDesc.java |   8 +-
 ...t_kylin_cube_without_slr_left_join_desc.json |   3 +-
 .../kylin/rest/controller/CubeController.java   |   2 +-
 .../source/hive/CreateFlatHiveTableStep.java|  32 +++-
 .../apache/kylin/source/hive/HiveMRInput.java   | 169 ++-
 10 files changed, 234 insertions(+), 37 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/7c5c6c1c/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 2ac9d48..de9051c 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -805,7 +805,7 @@ abstract public class KylinConfigBase implements 
Serializable {
 setProperty("kylin.dict.append.cache.size", String.valueOf(cacheSize));
 }
 
-public boolean getTableJoinTypeCheck() {
-return 
Boolean.valueOf(this.getOptional("kylin.table.join.strong.check", "true"));
+public String getCreateFlatHiveTableMethod() {
+return getOptional("kylin.hive.create.flat.table.method", "1");
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/7c5c6c1c/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
--
diff --git a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java 
b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
index b39265d..699d084 100644
--- a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
+++ b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
@@ -107,14 +107,14 @@ public class JoinedFlatTable {
 return ddl.toString();
 }
 
-public static String generateInsertDataStatement(IJoinedFlatTableDesc 
intermediateTableDesc, JobEngineConfig engineConfig) {
+public static String generateInsertDataStatement(IJoinedFlatTableDesc 
intermediateTableDesc, JobEngineConfig engineConfig, boolean redistribute) {
 StringBuilder sql = new StringBuilder();
 sql.append(generateHiveSetStatements(engineConfig));
-sql.append("INSERT OVERWRITE TABLE " + 
intermediateTableDesc.getTableName() + " " + 
generateSelectDataStatement(intermediateTableDesc) + ";").append("\n");
+sql.append("INSERT OVERWRITE TABLE " + 
intermediateTableDesc.getTableName() + " " + 
generateSelectDataStatement(intermediateTableDesc, redistribute) + 
";").append("\n");
 return sql.toString();
 }
 
-public static String generateSelectDataStatement(IJoinedFlatTableDesc 
flatDesc) {
+public static String generateSelectDataStatement(IJoinedFlatTableDesc 
flatDesc, boolean redistribute) {
 StringBuilder sql = new StringBuilder();
 sql.append("SELECT" + "\n");
 String tableAlias;
@@ -129,7 +129,15 @@ public class JoinedFlatTable {
 }
 appendJoinStatement(flatDesc, sql, tableAliasMap);
 appendWhereStatement(flatDesc, sql, tableAliasMap);
-appendDistributeStatement(flatDesc, sql, tableAliasMap);
+if (redistribute == true) {
+String redistributeCol = null;
+TblColRef distDcol = flatDesc.getDistributedBy();
+if (distDcol != null) {
+String tblAlias = tableAliasMap.get(distDcol.getTable());
+redistributeCol = tblAlias + "." + distDcol.getName();
+}
+appendDistributeStatement(sql, redistributeCol);
+}
 return sql.toString();
 }
 
@@ -228,14 +236,11 @@ public class JoinedFlatTable {
 return result;
 }
 
-private

kylin git commit: minor, add diagnosis to smoke test

2016-09-09 Thread lidong 
Repository: kylin
Updated Branches:
  refs/heads/1.5.4-rc1 d61bcbfd8 -> 639e4b015


minor, add diagnosis to smoke test


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/639e4b01
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/639e4b01
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/639e4b01

Branch: refs/heads/1.5.4-rc1
Commit: 639e4b015bab8bd9791f91257c3861e8ca1b1ef2
Parents: d61bcbf
Author: lidongsjtu 
Authored: Fri Sep 9 12:37:05 2016 +0800
Committer: lidongsjtu 
Committed: Fri Sep 9 18:55:43 2016 +0800

--
 build/smoke-test/smoke-test.sh |  1 +
 build/smoke-test/testDiag.py   | 44 +
 2 files changed, 45 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/639e4b01/build/smoke-test/smoke-test.sh
--
diff --git a/build/smoke-test/smoke-test.sh b/build/smoke-test/smoke-test.sh
index 06d930a..c21bd6d 100755
--- a/build/smoke-test/smoke-test.sh
+++ b/build/smoke-test/smoke-test.sh
@@ -67,6 +67,7 @@ sleep 3m
 cd $dir/smoke-test
 python testBuildCube.py || { exit 1; }
 python testQuery.py || { exit 1; }
+python testDiag.py || { exit 1; }
 cd -
 
 # Tear down stage

http://git-wip-us.apache.org/repos/asf/kylin/blob/639e4b01/build/smoke-test/testDiag.py
--
diff --git a/build/smoke-test/testDiag.py b/build/smoke-test/testDiag.py
new file mode 100644
index 000..cc932da
--- /dev/null
+++ b/build/smoke-test/testDiag.py
@@ -0,0 +1,44 @@
+#!/usr/bin/python
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# This is python unittest used in smoke-test.sh, aim to testing query via rest 
APIs.
+
+import unittest
+import requests
+
+class testDiag(unittest.TestCase):
+def setUp(self):
+pass
+
+def tearDown(self):
+pass
+
+def testDiag(self):
+url = "http://sandbox:7070/kylin/api/diag/project/learn_kylin/download;
+headers = {
+'content-type': "application/json",
+'authorization': "Basic QURNSU46S1lMSU4=",
+'cache-control': "no-cache"
+}
+
+response = requests.get(url, headers = headers)
+self.assertEqual(response.status_code, 200, 'Diagnosis failed.')
+
+
+if __name__ == '__main__':
+print 'Test Diagnogis for Kylin sample.'
+unittest.main()

[2/5] kylin git commit: KYLIN-2005 Move all storage side behavior hints to GTScanRequest

2016-09-09 Thread mahongbin 
KYLIN-2005 Move all storage side behavior hints to GTScanRequest


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d61bcbfd
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d61bcbfd
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d61bcbfd

Branch: refs/heads/1.5.4-rc1
Commit: d61bcbfd8bbe50d7cd08721234d9434f1140c861
Parents: dcd47f8
Author: Hongbin Ma 
Authored: Fri Sep 9 16:46:22 2016 +0800
Committer: Hongbin Ma 
Committed: Fri Sep 9 18:52:49 2016 +0800

--
 .../apache/kylin/gridtable/GTScanRequest.java   |  33 +-
 .../kylin/gridtable/GTScanRequestBuilder.java   |  30 +-
 .../kylin/gridtable/StorageSideBehavior.java|  30 +
 .../apache/kylin/query/ITKylinQueryTest.java|   4 +-
 .../common/coprocessor/CoprocessorBehavior.java |  30 -
 .../observer/AggregateRegionObserver.java   |  10 +-
 .../observer/AggregationScanner.java|  16 +-
 .../coprocessor/observer/ObserverEnabler.java   |   6 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java |  88 +--
 .../hbase/cube/v2/ExpectedSizeIterator.java |   4 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  18 +-
 .../endpoint/generated/CubeVisitProtos.java | 754 ---
 .../endpoint/protobuf/CubeVisit.proto   |  13 +-
 .../observer/AggregateRegionObserverTest.java   |   6 +-
 14 files changed, 332 insertions(+), 710 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/d61bcbfd/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java 
b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
index 5d27028..3e57e86 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
@@ -59,6 +59,9 @@ public class GTScanRequest {
 private String[] aggrMetricsFuncs;//
 
 // hint to storage behavior
+private String storageBehavior;
+private long startTime;
+private long timeout;
 private boolean allowStorageAggregation;
 private double aggCacheMemThreshold;
 private int storageScanRowNumThreshold;
@@ -69,7 +72,7 @@ public class GTScanRequest {
 
 GTScanRequest(GTInfo info, List ranges, ImmutableBitSet 
dimensions, ImmutableBitSet aggrGroupBy, //
 ImmutableBitSet aggrMetrics, String[] aggrMetricsFuncs, 
TupleFilter filterPushDown, boolean allowStorageAggregation, //
-double aggCacheMemThreshold, int storageScanRowNumThreshold, int 
storagePushDownLimit) {
+double aggCacheMemThreshold, int storageScanRowNumThreshold, int 
storagePushDownLimit, String storageBehavior, long startTime, long timeout) {
 this.info = info;
 if (ranges == null) {
 this.ranges = Lists.newArrayList(new GTScanRange(new 
GTRecord(info), new GTRecord(info)));
@@ -83,6 +86,9 @@ public class GTScanRequest {
 this.aggrMetrics = aggrMetrics;
 this.aggrMetricsFuncs = aggrMetricsFuncs;
 
+this.storageBehavior = storageBehavior;
+this.startTime = startTime;
+this.timeout = timeout;
 this.allowStorageAggregation = allowStorageAggregation;
 this.aggCacheMemThreshold = aggCacheMemThreshold;
 this.storageScanRowNumThreshold = storageScanRowNumThreshold;
@@ -115,6 +121,10 @@ public class GTScanRequest {
 }
 }
 
+public void setTimeout(long timeout) {
+this.timeout = timeout;
+}
+
 private void validateFilterPushDown(GTInfo info) {
 if (!hasFilterPushDown())
 return;
@@ -280,6 +290,18 @@ public class GTScanRequest {
 return this.storagePushDownLimit;
 }
 
+public String getStorageBehavior() {
+return storageBehavior;
+}
+
+public long getStartTime() {
+return startTime;
+}
+
+public long getTimeout() {
+return timeout;
+}
+
 @Override
 public String toString() {
 return "GTScanRequest [range=" + ranges + ", columns=" + columns + ", 
filterPushDown=" + filterPushDown + ", aggrGroupBy=" + aggrGroupBy + ", 
aggrMetrics=" + aggrMetrics + ", aggrMetricsFuncs=" + 
Arrays.toString(aggrMetricsFuncs) + "]";
@@ -320,6 +342,9 @@ public class GTScanRequest {
 out.putDouble(value.aggCacheMemThreshold);
 BytesUtil.writeVInt(value.storageScanRowNumThreshold, out);
 BytesUtil.writeVInt(value.storagePushDownLimit, out);
+BytesUtil.writeVLong(value.startTime, out);
+BytesUtil.writeVLong(value.timeout, out);
+BytesUtil.writeUTFString(value.storageBehavior, out);
 }

[4/5] kylin git commit: KYLIN-1922 optimize needStorageAggregation check logic and make sure self-termination in coprocessor works

2016-09-09 Thread mahongbin 
KYLIN-1922 optimize needStorageAggregation check logic and make sure 
self-termination in coprocessor works


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/dcd47f86
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/dcd47f86
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/dcd47f86

Branch: refs/heads/1.5.4-rc1
Commit: dcd47f86b6119f61670070750970797b12a3b6ed
Parents: 1df35ab
Author: Hongbin Ma 
Authored: Fri Sep 9 15:57:25 2016 +0800
Committer: Hongbin Ma 
Committed: Fri Sep 9 18:52:49 2016 +0800

--
 .../apache/kylin/common/KylinConfigBase.java|  15 +-
 .../apache/kylin/cube/RawQueryLastHacker.java   |   7 +-
 .../cube/gridtable/CubeScanRangePlanner.java| 340 --
 .../kylin/gridtable/GTAggregateScanner.java |  10 +-
 .../apache/kylin/gridtable/GTFilterScanner.java |   6 +-
 .../GTScanExceedThresholdException.java |   2 +-
 .../apache/kylin/gridtable/GTScanRequest.java   |  34 +-
 .../GTScanSelfTerminatedException.java  |  26 +
 .../kylin/gridtable/GTScanTimeoutException.java |   2 +-
 .../gridtable/AggregationCacheSpillTest.java|   6 +-
 .../kylin/gridtable/DictGridTableTest.java  | 617 --
 .../storage/gtrecord/CubeScanRangePlanner.java  | 357 +++
 .../storage/gtrecord/CubeSegmentScanner.java|  14 +-
 .../gtrecord/GTCubeStorageQueryBase.java|  36 +-
 .../storage/gtrecord/DictGridTableTest.java | 626 +++
 .../apache/kylin/query/ITKylinQueryTest.java|  55 +-
 .../resources/query/sql_timeout/query01.sql |  19 +
 .../common/coprocessor/CoprocessorBehavior.java |   1 +
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java |   8 +-
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |   2 +-
 .../hbase/cube/v2/ExpectedSizeIterator.java |  39 +-
 .../hbase/cube/v2/HBaseReadonlyStore.java   |  11 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  26 +-
 23 files changed, 1197 insertions(+), 1062 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/dcd47f86/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index f0c91da..2ac9d48 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -481,8 +481,8 @@ abstract public class KylinConfigBase implements 
Serializable {
 return Integer.parseInt(getOptional("kylin.query.scan.threshold", 
"1000"));
 }
 
-public int getCubeVisitTimeoutTimes() {
-return 
Integer.parseInt(getOptional("kylin.query.cube.visit.timeout.times", "1"));
+public float getCubeVisitTimeoutTimes() {
+return 
Float.parseFloat(getOptional("kylin.query.cube.visit.timeout.times", "1"));
 }
 
 public int getBadQueryStackTraceDepth() {
@@ -545,15 +545,6 @@ abstract public class KylinConfigBase implements 
Serializable {
 return 
Boolean.parseBoolean(this.getOptional("kylin.query.ignore_unknown_function", 
"false"));
 }
 
-public String getQueryStorageVisitPlanner() {
-return this.getOptional("kylin.query.storage.visit.planner", 
"org.apache.kylin.cube.gridtable.CubeScanRangePlanner");
-}
-
-// for test only
-public void setQueryStorageVisitPlanner(String v) {
-setProperty("kylin.query.storage.visit.planner", v);
-}
-
 public int getQueryScanFuzzyKeyMax() {
 return 
Integer.parseInt(this.getOptional("kylin.query.scan.fuzzykey.max", "200"));
 }
@@ -573,7 +564,7 @@ abstract public class KylinConfigBase implements 
Serializable {
 public boolean getQueryMetricsEnabled() {
 return Boolean.parseBoolean(getOptional("kylin.query.metrics.enabled", 
"false"));
 }
-
+
 public int[] getQueryMetricsPercentilesIntervals() {
 String[] dft = { "60", "300", "3600" };
 return 
getOptionalIntArray("kylin.query.metrics.percentiles.intervals", dft);

http://git-wip-us.apache.org/repos/asf/kylin/blob/dcd47f86/core-cube/src/main/java/org/apache/kylin/cube/RawQueryLastHacker.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/RawQueryLastHacker.java 
b/core-cube/src/main/java/org/apache/kylin/cube/RawQueryLastHacker.java
index 63ddac5..50c644e 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/RawQueryLastHacker.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/RawQueryLastHacker.java
@@ -44,13 +44,14 @@ public class RawQueryLastHacker {

[5/5] kylin git commit: minor, add one more test case

2016-09-09 Thread mahongbin 
minor, add one more test case


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1df35ab8
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1df35ab8
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1df35ab8

Branch: refs/heads/1.5.4-rc1
Commit: 1df35ab852c21e8dc6c90f82d343a5432661db06
Parents: 6ddf4b5
Author: Hongbin Ma 
Authored: Thu Sep 8 11:18:52 2016 +0800
Committer: Hongbin Ma 
Committed: Fri Sep 9 18:52:49 2016 +0800

--
 .../org/apache/kylin/cube/CubeCapabilityChecker.java   |  7 ++-
 kylin-it/src/test/resources/query/sql_like/query18.sql | 13 +
 2 files changed, 19 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/1df35ab8/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
index 79d1e3b..e8c96b4 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
@@ -31,6 +31,7 @@ import org.apache.kylin.metadata.filter.UDF.MassInTupleFilter;
 import org.apache.kylin.metadata.model.FunctionDesc;
 import org.apache.kylin.metadata.model.IStorageAware;
 import org.apache.kylin.metadata.model.MeasureDesc;
+import org.apache.kylin.metadata.model.ParameterDesc;
 import org.apache.kylin.metadata.model.TblColRef;
 import org.apache.kylin.metadata.realization.CapabilityResult;
 import org.apache.kylin.metadata.realization.SQLDigest;
@@ -146,7 +147,11 @@ public class CubeCapabilityChecker {
 }
 
 // calcite can do aggregation from columns on-the-fly
-List neededCols = 
functionDesc.getParameter().getColRefs();
+ParameterDesc parameterDesc = functionDesc.getParameter();
+if (parameterDesc == null) {
+continue;
+}
+List neededCols = parameterDesc.getColRefs();
 if (neededCols.size() > 0 && 
cubeDesc.listDimensionColumnsIncludingDerived().containsAll(neededCols) && 
FunctionDesc.BUILT_IN_AGGREGATIONS.contains(functionDesc.getExpression())) {
 result.influences.add(new 
CapabilityResult.DimensionAsMeasure(functionDesc));
 it.remove();

http://git-wip-us.apache.org/repos/asf/kylin/blob/1df35ab8/kylin-it/src/test/resources/query/sql_like/query18.sql
--
diff --git a/kylin-it/src/test/resources/query/sql_like/query18.sql 
b/kylin-it/src/test/resources/query/sql_like/query18.sql
new file mode 100644
index 000..8ef6ad4
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_like/query18.sql
@@ -0,0 +1,13 @@
+
+select USER_DEFINED_FIELD3 as abc
+ 
+ from test_kylin_fact 
+inner JOIN edw.test_cal_dt as test_cal_dt
+ ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
+ inner JOIN test_category_groupings
+ ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND 
test_kylin_fact.lstg_site_id = test_category_groupings.site_id
+ inner JOIN edw.test_sites as test_sites
+ ON test_kylin_fact.lstg_site_id = test_sites.site_id
+ 
+ 
+where upper(USER_DEFINED_FIELD3) like '%VID%'

[1/5] kylin git commit: KYLIN-2005 Move all storage side behavior hints to GTScanRequest

2016-09-09 Thread mahongbin 
Repository: kylin
Updated Branches:
  refs/heads/1.5.4-rc1 6ddf4b5f9 -> d61bcbfd8


http://git-wip-us.apache.org/repos/asf/kylin/blob/d61bcbfd/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/generated/CubeVisitProtos.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/generated/CubeVisitProtos.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/generated/CubeVisitProtos.java
index d9cef88..b0688b7 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/generated/CubeVisitProtos.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/generated/CubeVisitProtos.java
@@ -11,115 +11,64 @@ public final class CubeVisitProtos {
   public interface CubeVisitRequestOrBuilder
   extends com.google.protobuf.MessageOrBuilder {
 
-// required string behavior = 1;
+// required bytes gtScanRequest = 1;
 /**
- * required string behavior = 1;
- */
-boolean hasBehavior();
-/**
- * required string behavior = 1;
- */
-java.lang.String getBehavior();
-/**
- * required string behavior = 1;
- */
-com.google.protobuf.ByteString
-getBehaviorBytes();
-
-// required bytes gtScanRequest = 2;
-/**
- * required bytes gtScanRequest = 2;
+ * required bytes gtScanRequest = 1;
  */
 boolean hasGtScanRequest();
 /**
- * required bytes gtScanRequest = 2;
+ * required bytes gtScanRequest = 1;
  */
 com.google.protobuf.ByteString getGtScanRequest();
 
-// required bytes hbaseRawScan = 3;
+// required bytes hbaseRawScan = 2;
 /**
- * required bytes hbaseRawScan = 3;
+ * required bytes hbaseRawScan = 2;
  */
 boolean hasHbaseRawScan();
 /**
- * required bytes hbaseRawScan = 3;
+ * required bytes hbaseRawScan = 2;
  */
 com.google.protobuf.ByteString getHbaseRawScan();
 
-// required int32 rowkeyPreambleSize = 4;
+// required int32 rowkeyPreambleSize = 3;
 /**
- * required int32 rowkeyPreambleSize = 4;
+ * required int32 rowkeyPreambleSize = 3;
  */
 boolean hasRowkeyPreambleSize();
 /**
- * required int32 rowkeyPreambleSize = 4;
+ * required int32 rowkeyPreambleSize = 3;
  */
 int getRowkeyPreambleSize();
 
-// repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 5;
+// repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;
 /**
- * repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 5;
+ * repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;
  */
 
java.util.List
 
 getHbaseColumnsToGTList();
 /**
- * repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 5;
+ * repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;
  */
 
org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList
 getHbaseColumnsToGT(int index);
 /**
- * repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 5;
+ * repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;
  */
 int getHbaseColumnsToGTCount();
 /**
- * repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 5;
+ * repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;
  */
 java.util.List
 
 getHbaseColumnsToGTOrBuilderList();
 /**
- * repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 5;
+ * repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;
  */
 
org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntListOrBuilder
 getHbaseColumnsToGTOrBuilder(
 int index);
 
-// required int64 startTime = 6;
-/**
- * required int64 startTime = 6;
- *
- * 
- *when client start the request
- * 
- */
-boolean hasStartTime();
-/**
- * required int64 startTime = 6;
- *
- * 
- *when client start the request
- * 
- */
-long getStartTime();
-
-// required int64 timeout = 7;
+// required string kylinProperties = 5;
 /**
- * required int64 timeout = 7;
- *
- * 
- *how long client will wait
- * 
- */
-boolean hasTimeout();
-/**
- * required int64 timeout = 7;
- *
- * 
- *how long client will wait
- * 
- */
-long getTimeout();
-
-// required string kylinProperties = 8;
-/**
- * required string kylinProperties = 8;
+ * required string kylinProperties = 5;
  *
  * 
  * kylin properties
@@ -127,7 +76,7 @@ public final class CubeVisitProtos {
  */
 boolean hasKylinProperties();
 /**
- * required string kylinProperties = 8;
+ * required string kylinProperties = 5;
  *
  * 
  * kylin properties
@@

[3/5] kylin git commit: KYLIN-1922 optimize needStorageAggregation check logic and make sure self-termination in coprocessor works

2016-09-09 Thread mahongbin 
http://git-wip-us.apache.org/repos/asf/kylin/blob/dcd47f86/core-storage/src/test/java/org/apache/kylin/storage/gtrecord/DictGridTableTest.java
--
diff --git 
a/core-storage/src/test/java/org/apache/kylin/storage/gtrecord/DictGridTableTest.java
 
b/core-storage/src/test/java/org/apache/kylin/storage/gtrecord/DictGridTableTest.java
new file mode 100644
index 000..0cdfa7e
--- /dev/null
+++ 
b/core-storage/src/test/java/org/apache/kylin/storage/gtrecord/DictGridTableTest.java
@@ -0,0 +1,626 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *  
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *  
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.storage.gtrecord;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.BitSet;
+import java.util.List;
+
+import org.apache.kylin.common.util.ByteArray;
+import org.apache.kylin.common.util.BytesSerializer;
+import org.apache.kylin.common.util.Dictionary;
+import org.apache.kylin.common.util.ImmutableBitSet;
+import org.apache.kylin.common.util.LocalFileMetadataTestCase;
+import org.apache.kylin.common.util.Pair;
+import org.apache.kylin.cube.gridtable.CubeCodeSystem;
+import org.apache.kylin.dict.NumberDictionaryBuilder;
+import org.apache.kylin.dict.StringBytesConverter;
+import org.apache.kylin.dict.TrieDictionaryBuilder;
+import org.apache.kylin.dimension.DictionaryDimEnc;
+import org.apache.kylin.dimension.DimensionEncoding;
+import org.apache.kylin.gridtable.GTBuilder;
+import org.apache.kylin.gridtable.GTInfo;
+import org.apache.kylin.gridtable.GTRecord;
+import org.apache.kylin.gridtable.GTScanRange;
+import org.apache.kylin.gridtable.GTScanRequest;
+import org.apache.kylin.gridtable.GTScanRequestBuilder;
+import org.apache.kylin.gridtable.GTUtil;
+import org.apache.kylin.gridtable.GridTable;
+import org.apache.kylin.gridtable.IGTScanner;
+import org.apache.kylin.gridtable.GTFilterScanner.FilterResultCache;
+import org.apache.kylin.gridtable.GTInfo.Builder;
+import org.apache.kylin.gridtable.memstore.GTSimpleMemStore;
+import org.apache.kylin.metadata.datatype.DataType;
+import org.apache.kylin.metadata.datatype.LongMutable;
+import org.apache.kylin.metadata.filter.ColumnTupleFilter;
+import org.apache.kylin.metadata.filter.CompareTupleFilter;
+import org.apache.kylin.metadata.filter.ConstantTupleFilter;
+import org.apache.kylin.metadata.filter.ExtractTupleFilter;
+import org.apache.kylin.metadata.filter.LogicalTupleFilter;
+import org.apache.kylin.metadata.filter.TupleFilter;
+import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
+import org.apache.kylin.metadata.model.ColumnDesc;
+import org.apache.kylin.metadata.model.TableDesc;
+import org.apache.kylin.metadata.model.TblColRef;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.google.common.collect.Lists;
+
+public class DictGridTableTest extends LocalFileMetadataTestCase {
+
+private GridTable table;
+private GTInfo info;
+private CompareTupleFilter timeComp0;
+private CompareTupleFilter timeComp1;
+private CompareTupleFilter timeComp2;
+private CompareTupleFilter timeComp3;
+private CompareTupleFilter timeComp4;
+private CompareTupleFilter timeComp5;
+private CompareTupleFilter timeComp6;
+private CompareTupleFilter ageComp1;
+private CompareTupleFilter ageComp2;
+private CompareTupleFilter ageComp3;
+private CompareTupleFilter ageComp4;
+
+@After
+public void after() throws Exception {
+
+this.cleanupTestMetadata();
+}
+
+@Before
+public void setup() throws IOException {
+
+this.createTestMetadata();
+
+table = newTestTable();
+info = table.getInfo();
+
+timeComp0 = compare(info.colRef(0), FilterOperatorEnum.LT, enc(info, 
0, "2015-01-14"));
+timeComp1 = compare(info.colRef(0), FilterOperatorEnum.GT, enc(info, 
0, "2015-01-14"));
+timeComp2 = compare(info.colRef(0), FilterOperatorEnum.LT, enc(info, 
0, "2015-01-13"));
+timeComp3 = compare(info.colRef(0), FilterOperatorEnum.LT, enc(info, 
0,

[1/4] kylin git commit: KYLIN-1922 optimize needStorageAggregation check logic and make sure self-termination in coprocessor works

2016-09-09 Thread mahongbin 
Repository: kylin
Updated Branches:
  refs/heads/master e87c816dd -> a2c875d8a


http://git-wip-us.apache.org/repos/asf/kylin/blob/e38557b4/core-storage/src/test/java/org/apache/kylin/storage/gtrecord/DictGridTableTest.java
--
diff --git 
a/core-storage/src/test/java/org/apache/kylin/storage/gtrecord/DictGridTableTest.java
 
b/core-storage/src/test/java/org/apache/kylin/storage/gtrecord/DictGridTableTest.java
new file mode 100644
index 000..0cdfa7e
--- /dev/null
+++ 
b/core-storage/src/test/java/org/apache/kylin/storage/gtrecord/DictGridTableTest.java
@@ -0,0 +1,626 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *  
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *  
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.storage.gtrecord;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.BitSet;
+import java.util.List;
+
+import org.apache.kylin.common.util.ByteArray;
+import org.apache.kylin.common.util.BytesSerializer;
+import org.apache.kylin.common.util.Dictionary;
+import org.apache.kylin.common.util.ImmutableBitSet;
+import org.apache.kylin.common.util.LocalFileMetadataTestCase;
+import org.apache.kylin.common.util.Pair;
+import org.apache.kylin.cube.gridtable.CubeCodeSystem;
+import org.apache.kylin.dict.NumberDictionaryBuilder;
+import org.apache.kylin.dict.StringBytesConverter;
+import org.apache.kylin.dict.TrieDictionaryBuilder;
+import org.apache.kylin.dimension.DictionaryDimEnc;
+import org.apache.kylin.dimension.DimensionEncoding;
+import org.apache.kylin.gridtable.GTBuilder;
+import org.apache.kylin.gridtable.GTInfo;
+import org.apache.kylin.gridtable.GTRecord;
+import org.apache.kylin.gridtable.GTScanRange;
+import org.apache.kylin.gridtable.GTScanRequest;
+import org.apache.kylin.gridtable.GTScanRequestBuilder;
+import org.apache.kylin.gridtable.GTUtil;
+import org.apache.kylin.gridtable.GridTable;
+import org.apache.kylin.gridtable.IGTScanner;
+import org.apache.kylin.gridtable.GTFilterScanner.FilterResultCache;
+import org.apache.kylin.gridtable.GTInfo.Builder;
+import org.apache.kylin.gridtable.memstore.GTSimpleMemStore;
+import org.apache.kylin.metadata.datatype.DataType;
+import org.apache.kylin.metadata.datatype.LongMutable;
+import org.apache.kylin.metadata.filter.ColumnTupleFilter;
+import org.apache.kylin.metadata.filter.CompareTupleFilter;
+import org.apache.kylin.metadata.filter.ConstantTupleFilter;
+import org.apache.kylin.metadata.filter.ExtractTupleFilter;
+import org.apache.kylin.metadata.filter.LogicalTupleFilter;
+import org.apache.kylin.metadata.filter.TupleFilter;
+import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
+import org.apache.kylin.metadata.model.ColumnDesc;
+import org.apache.kylin.metadata.model.TableDesc;
+import org.apache.kylin.metadata.model.TblColRef;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.google.common.collect.Lists;
+
+public class DictGridTableTest extends LocalFileMetadataTestCase {
+
+private GridTable table;
+private GTInfo info;
+private CompareTupleFilter timeComp0;
+private CompareTupleFilter timeComp1;
+private CompareTupleFilter timeComp2;
+private CompareTupleFilter timeComp3;
+private CompareTupleFilter timeComp4;
+private CompareTupleFilter timeComp5;
+private CompareTupleFilter timeComp6;
+private CompareTupleFilter ageComp1;
+private CompareTupleFilter ageComp2;
+private CompareTupleFilter ageComp3;
+private CompareTupleFilter ageComp4;
+
+@After
+public void after() throws Exception {
+
+this.cleanupTestMetadata();
+}
+
+@Before
+public void setup() throws IOException {
+
+this.createTestMetadata();
+
+table = newTestTable();
+info = table.getInfo();
+
+timeComp0 = compare(info.colRef(0), FilterOperatorEnum.LT, enc(info, 
0, "2015-01-14"));
+timeComp1 = compare(info.colRef(0), FilterOperatorEnum.GT, enc(info, 
0, "2015-01-14"));
+timeComp2 = compare(info.colRef(0), FilterOperatorEnum.LT, enc(info, 
0, "2015-01-13"));
+

[2/4] kylin git commit: KYLIN-1922 optimize needStorageAggregation check logic and make sure self-termination in coprocessor works

2016-09-09 Thread mahongbin 
KYLIN-1922 optimize needStorageAggregation check logic and make sure 
self-termination in coprocessor works


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e38557b4
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e38557b4
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e38557b4

Branch: refs/heads/master
Commit: e38557b4d1cd1d42fe042e5500020cbfaba2d80b
Parents: e87c816
Author: Hongbin Ma 
Authored: Fri Sep 9 15:57:25 2016 +0800
Committer: Hongbin Ma 
Committed: Fri Sep 9 16:42:33 2016 +0800

--
 .../apache/kylin/common/KylinConfigBase.java|  15 +-
 .../apache/kylin/cube/RawQueryLastHacker.java   |   7 +-
 .../cube/gridtable/CubeScanRangePlanner.java| 340 --
 .../kylin/gridtable/GTAggregateScanner.java |  10 +-
 .../apache/kylin/gridtable/GTFilterScanner.java |   6 +-
 .../GTScanExceedThresholdException.java |   2 +-
 .../apache/kylin/gridtable/GTScanRequest.java   |  34 +-
 .../GTScanSelfTerminatedException.java  |  26 +
 .../kylin/gridtable/GTScanTimeoutException.java |   2 +-
 .../gridtable/AggregationCacheSpillTest.java|   6 +-
 .../kylin/gridtable/DictGridTableTest.java  | 617 --
 .../storage/gtrecord/CubeScanRangePlanner.java  | 357 +++
 .../storage/gtrecord/CubeSegmentScanner.java|  14 +-
 .../gtrecord/GTCubeStorageQueryBase.java|  36 +-
 .../storage/gtrecord/DictGridTableTest.java | 626 +++
 .../apache/kylin/query/ITKylinQueryTest.java|  55 +-
 .../resources/query/sql_timeout/query01.sql |  19 +
 .../common/coprocessor/CoprocessorBehavior.java |   1 +
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java |   8 +-
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |   2 +-
 .../hbase/cube/v2/ExpectedSizeIterator.java |  39 +-
 .../hbase/cube/v2/HBaseReadonlyStore.java   |  11 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  26 +-
 23 files changed, 1197 insertions(+), 1062 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/e38557b4/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index f0c91da..2ac9d48 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -481,8 +481,8 @@ abstract public class KylinConfigBase implements 
Serializable {
 return Integer.parseInt(getOptional("kylin.query.scan.threshold", 
"1000"));
 }
 
-public int getCubeVisitTimeoutTimes() {
-return 
Integer.parseInt(getOptional("kylin.query.cube.visit.timeout.times", "1"));
+public float getCubeVisitTimeoutTimes() {
+return 
Float.parseFloat(getOptional("kylin.query.cube.visit.timeout.times", "1"));
 }
 
 public int getBadQueryStackTraceDepth() {
@@ -545,15 +545,6 @@ abstract public class KylinConfigBase implements 
Serializable {
 return 
Boolean.parseBoolean(this.getOptional("kylin.query.ignore_unknown_function", 
"false"));
 }
 
-public String getQueryStorageVisitPlanner() {
-return this.getOptional("kylin.query.storage.visit.planner", 
"org.apache.kylin.cube.gridtable.CubeScanRangePlanner");
-}
-
-// for test only
-public void setQueryStorageVisitPlanner(String v) {
-setProperty("kylin.query.storage.visit.planner", v);
-}
-
 public int getQueryScanFuzzyKeyMax() {
 return 
Integer.parseInt(this.getOptional("kylin.query.scan.fuzzykey.max", "200"));
 }
@@ -573,7 +564,7 @@ abstract public class KylinConfigBase implements 
Serializable {
 public boolean getQueryMetricsEnabled() {
 return Boolean.parseBoolean(getOptional("kylin.query.metrics.enabled", 
"false"));
 }
-
+
 public int[] getQueryMetricsPercentilesIntervals() {
 String[] dft = { "60", "300", "3600" };
 return 
getOptionalIntArray("kylin.query.metrics.percentiles.intervals", dft);

http://git-wip-us.apache.org/repos/asf/kylin/blob/e38557b4/core-cube/src/main/java/org/apache/kylin/cube/RawQueryLastHacker.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/RawQueryLastHacker.java 
b/core-cube/src/main/java/org/apache/kylin/cube/RawQueryLastHacker.java
index 63ddac5..50c644e 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/RawQueryLastHacker.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/RawQueryLastHacker.java
@@ -44,13 +44,14 @@ public class RawQueryLastHacker {
 //

[4/4] kylin git commit: KYLIN-2005 Move all storage side behavior hints to GTScanRequest

2016-09-09 Thread mahongbin 
KYLIN-2005 Move all storage side behavior hints to GTScanRequest


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a2c875d8
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a2c875d8
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a2c875d8

Branch: refs/heads/master
Commit: a2c875d8a2d06f23dd6467bbcc459bff82918295
Parents: e38557b
Author: Hongbin Ma 
Authored: Fri Sep 9 16:46:22 2016 +0800
Committer: Hongbin Ma 
Committed: Fri Sep 9 17:47:29 2016 +0800

--
 .../apache/kylin/gridtable/GTScanRequest.java   |  33 +-
 .../kylin/gridtable/GTScanRequestBuilder.java   |  30 +-
 .../kylin/gridtable/StorageSideBehavior.java|  30 +
 .../apache/kylin/query/ITKylinQueryTest.java|   4 +-
 .../common/coprocessor/CoprocessorBehavior.java |  30 -
 .../observer/AggregateRegionObserver.java   |  10 +-
 .../observer/AggregationScanner.java|  16 +-
 .../coprocessor/observer/ObserverEnabler.java   |   6 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java |  88 +--
 .../hbase/cube/v2/ExpectedSizeIterator.java |   4 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  18 +-
 .../endpoint/generated/CubeVisitProtos.java | 754 ---
 .../endpoint/protobuf/CubeVisit.proto   |  13 +-
 .../observer/AggregateRegionObserverTest.java   |   6 +-
 14 files changed, 332 insertions(+), 710 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/a2c875d8/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java 
b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
index 5d27028..3e57e86 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
@@ -59,6 +59,9 @@ public class GTScanRequest {
 private String[] aggrMetricsFuncs;//
 
 // hint to storage behavior
+private String storageBehavior;
+private long startTime;
+private long timeout;
 private boolean allowStorageAggregation;
 private double aggCacheMemThreshold;
 private int storageScanRowNumThreshold;
@@ -69,7 +72,7 @@ public class GTScanRequest {
 
 GTScanRequest(GTInfo info, List ranges, ImmutableBitSet 
dimensions, ImmutableBitSet aggrGroupBy, //
 ImmutableBitSet aggrMetrics, String[] aggrMetricsFuncs, 
TupleFilter filterPushDown, boolean allowStorageAggregation, //
-double aggCacheMemThreshold, int storageScanRowNumThreshold, int 
storagePushDownLimit) {
+double aggCacheMemThreshold, int storageScanRowNumThreshold, int 
storagePushDownLimit, String storageBehavior, long startTime, long timeout) {
 this.info = info;
 if (ranges == null) {
 this.ranges = Lists.newArrayList(new GTScanRange(new 
GTRecord(info), new GTRecord(info)));
@@ -83,6 +86,9 @@ public class GTScanRequest {
 this.aggrMetrics = aggrMetrics;
 this.aggrMetricsFuncs = aggrMetricsFuncs;
 
+this.storageBehavior = storageBehavior;
+this.startTime = startTime;
+this.timeout = timeout;
 this.allowStorageAggregation = allowStorageAggregation;
 this.aggCacheMemThreshold = aggCacheMemThreshold;
 this.storageScanRowNumThreshold = storageScanRowNumThreshold;
@@ -115,6 +121,10 @@ public class GTScanRequest {
 }
 }
 
+public void setTimeout(long timeout) {
+this.timeout = timeout;
+}
+
 private void validateFilterPushDown(GTInfo info) {
 if (!hasFilterPushDown())
 return;
@@ -280,6 +290,18 @@ public class GTScanRequest {
 return this.storagePushDownLimit;
 }
 
+public String getStorageBehavior() {
+return storageBehavior;
+}
+
+public long getStartTime() {
+return startTime;
+}
+
+public long getTimeout() {
+return timeout;
+}
+
 @Override
 public String toString() {
 return "GTScanRequest [range=" + ranges + ", columns=" + columns + ", 
filterPushDown=" + filterPushDown + ", aggrGroupBy=" + aggrGroupBy + ", 
aggrMetrics=" + aggrMetrics + ", aggrMetricsFuncs=" + 
Arrays.toString(aggrMetricsFuncs) + "]";
@@ -320,6 +342,9 @@ public class GTScanRequest {
 out.putDouble(value.aggCacheMemThreshold);
 BytesUtil.writeVInt(value.storageScanRowNumThreshold, out);
 BytesUtil.writeVInt(value.storagePushDownLimit, out);
+BytesUtil.writeVLong(value.startTime, out);
+BytesUtil.writeVLong(value.timeout, out);
+BytesUtil.writeUTFString(value.storageBehavior, out);
 }

[3/4] kylin git commit: KYLIN-2005 Move all storage side behavior hints to GTScanRequest

2016-09-09 Thread mahongbin 
http://git-wip-us.apache.org/repos/asf/kylin/blob/a2c875d8/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/generated/CubeVisitProtos.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/generated/CubeVisitProtos.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/generated/CubeVisitProtos.java
index d9cef88..b0688b7 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/generated/CubeVisitProtos.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/generated/CubeVisitProtos.java
@@ -11,115 +11,64 @@ public final class CubeVisitProtos {
   public interface CubeVisitRequestOrBuilder
   extends com.google.protobuf.MessageOrBuilder {
 
-// required string behavior = 1;
+// required bytes gtScanRequest = 1;
 /**
- * required string behavior = 1;
- */
-boolean hasBehavior();
-/**
- * required string behavior = 1;
- */
-java.lang.String getBehavior();
-/**
- * required string behavior = 1;
- */
-com.google.protobuf.ByteString
-getBehaviorBytes();
-
-// required bytes gtScanRequest = 2;
-/**
- * required bytes gtScanRequest = 2;
+ * required bytes gtScanRequest = 1;
  */
 boolean hasGtScanRequest();
 /**
- * required bytes gtScanRequest = 2;
+ * required bytes gtScanRequest = 1;
  */
 com.google.protobuf.ByteString getGtScanRequest();
 
-// required bytes hbaseRawScan = 3;
+// required bytes hbaseRawScan = 2;
 /**
- * required bytes hbaseRawScan = 3;
+ * required bytes hbaseRawScan = 2;
  */
 boolean hasHbaseRawScan();
 /**
- * required bytes hbaseRawScan = 3;
+ * required bytes hbaseRawScan = 2;
  */
 com.google.protobuf.ByteString getHbaseRawScan();
 
-// required int32 rowkeyPreambleSize = 4;
+// required int32 rowkeyPreambleSize = 3;
 /**
- * required int32 rowkeyPreambleSize = 4;
+ * required int32 rowkeyPreambleSize = 3;
  */
 boolean hasRowkeyPreambleSize();
 /**
- * required int32 rowkeyPreambleSize = 4;
+ * required int32 rowkeyPreambleSize = 3;
  */
 int getRowkeyPreambleSize();
 
-// repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 5;
+// repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;
 /**
- * repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 5;
+ * repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;
  */
 
java.util.List
 
 getHbaseColumnsToGTList();
 /**
- * repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 5;
+ * repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;
  */
 
org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList
 getHbaseColumnsToGT(int index);
 /**
- * repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 5;
+ * repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;
  */
 int getHbaseColumnsToGTCount();
 /**
- * repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 5;
+ * repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;
  */
 java.util.List
 
 getHbaseColumnsToGTOrBuilderList();
 /**
- * repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 5;
+ * repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;
  */
 
org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntListOrBuilder
 getHbaseColumnsToGTOrBuilder(
 int index);
 
-// required int64 startTime = 6;
-/**
- * required int64 startTime = 6;
- *
- * 
- *when client start the request
- * 
- */
-boolean hasStartTime();
-/**
- * required int64 startTime = 6;
- *
- * 
- *when client start the request
- * 
- */
-long getStartTime();
-
-// required int64 timeout = 7;
+// required string kylinProperties = 5;
 /**
- * required int64 timeout = 7;
- *
- * 
- *how long client will wait
- * 
- */
-boolean hasTimeout();
-/**
- * required int64 timeout = 7;
- *
- * 
- *how long client will wait
- * 
- */
-long getTimeout();
-
-// required string kylinProperties = 8;
-/**
- * required string kylinProperties = 8;
+ * required string kylinProperties = 5;
  *
  * 
  * kylin properties
@@ -127,7 +76,7 @@ public final class CubeVisitProtos {
  */
 boolean hasKylinProperties();
 /**
- * required string kylinProperties = 8;
+ * required string kylinProperties = 5;
  *
  * 
  * kylin properties
@@ -135,7 +84,7 @@ public final class CubeVisitProtos {
  */
 java.lang.String

kylin git commit: KYLIN-1996 Keep original column order when designing cube

2016-09-09 Thread zhongjian 
Repository: kylin
Updated Branches:
  refs/heads/master bf2611486 -> e87c816dd


KYLIN-1996 Keep original column order when designing cube

Signed-off-by: Jason 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e87c816d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e87c816d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e87c816d

Branch: refs/heads/master
Commit: e87c816dd7099290220ca09361f9ca04c36a317e
Parents: bf26114
Author: chenzhx <346839...@qq.com>
Authored: Fri Sep 9 10:55:36 2016 +0800
Committer: Jason 
Committed: Fri Sep 9 15:52:48 2016 +0800

--
 webapp/app/js/controllers/cubeDimensions.js | 10 ++
 1 file changed, 2 insertions(+), 8 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/e87c816d/webapp/app/js/controllers/cubeDimensions.js
--
diff --git a/webapp/app/js/controllers/cubeDimensions.js 
b/webapp/app/js/controllers/cubeDimensions.js
index 1663b5c..ab07451 100644
--- a/webapp/app/js/controllers/cubeDimensions.js
+++ b/webapp/app/js/controllers/cubeDimensions.js
@@ -67,20 +67,17 @@ KylinApp.controller('CubeDimensionsCtrl', function ($scope, 
$modal,MetaModel,cub
 var cols = $scope.getDimColumnsByTable(factTable);
 
 // Initialize selected available.
-var factAvailable = {};
 var factSelectAvailable = {};
 
 for (var i = 0; i < cols.length; i++) {
 cols[i].table = factTable;
 cols[i].isLookup = false;
 
-factAvailable[cols[i].name] = cols[i];
-
 // Default not selected and not disabled.
 factSelectAvailable[cols[i].name] = {selected: false, disabled: 
false};
 }
 
-$scope.availableColumns[factTable] = factAvailable;
+$scope.availableColumns[factTable] = cols;
 $scope.selectedColumns[factTable] = factSelectAvailable;
 $scope.availableTables.push(factTable);
 
@@ -91,20 +88,17 @@ KylinApp.controller('CubeDimensionsCtrl', function ($scope, 
$modal,MetaModel,cub
 var cols2 = $scope.getDimColumnsByTable(lookups[j].table);
 
 // Initialize selected available.
-var lookupAvailable = {};
 var lookupSelectAvailable = {};
 
 for (var k = 0; k < cols2.length; k++) {
 cols2[k].table = lookups[j].table;
 cols2[k].isLookup = true;
 
-lookupAvailable[cols2[k].name] = cols2[k];
-
 // Default not selected and not disabled.
 lookupSelectAvailable[cols2[k].name] = {selected: false, 
disabled: false};
 }
 
-$scope.availableColumns[lookups[j].table] = lookupAvailable;
+$scope.availableColumns[lookups[j].table] = cols2;
 $scope.selectedColumns[lookups[j].table] = lookupSelectAvailable;
 if($scope.availableTables.indexOf(lookups[j].table)==-1){
 $scope.availableTables.push(lookups[j].table);

kylin git commit: KYLIN-1996 Keep original column order when designing cube

2016-09-09 Thread zhongjian 
Repository: kylin
Updated Branches:
  refs/heads/1.5.4-rc1 81cc8ee7a -> 6ddf4b5f9


KYLIN-1996 Keep original column order when designing cube

Signed-off-by: Jason 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6ddf4b5f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6ddf4b5f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6ddf4b5f

Branch: refs/heads/1.5.4-rc1
Commit: 6ddf4b5f98a2c55529e12899fffeb6eec9622fbc
Parents: 81cc8ee
Author: chenzhx <346839...@qq.com>
Authored: Fri Sep 9 10:55:36 2016 +0800
Committer: Jason 
Committed: Fri Sep 9 15:53:21 2016 +0800

--
 webapp/app/js/controllers/cubeDimensions.js | 10 ++
 1 file changed, 2 insertions(+), 8 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/6ddf4b5f/webapp/app/js/controllers/cubeDimensions.js
--
diff --git a/webapp/app/js/controllers/cubeDimensions.js 
b/webapp/app/js/controllers/cubeDimensions.js
index 1663b5c..ab07451 100644
--- a/webapp/app/js/controllers/cubeDimensions.js
+++ b/webapp/app/js/controllers/cubeDimensions.js
@@ -67,20 +67,17 @@ KylinApp.controller('CubeDimensionsCtrl', function ($scope, 
$modal,MetaModel,cub
 var cols = $scope.getDimColumnsByTable(factTable);
 
 // Initialize selected available.
-var factAvailable = {};
 var factSelectAvailable = {};
 
 for (var i = 0; i < cols.length; i++) {
 cols[i].table = factTable;
 cols[i].isLookup = false;
 
-factAvailable[cols[i].name] = cols[i];
-
 // Default not selected and not disabled.
 factSelectAvailable[cols[i].name] = {selected: false, disabled: 
false};
 }
 
-$scope.availableColumns[factTable] = factAvailable;
+$scope.availableColumns[factTable] = cols;
 $scope.selectedColumns[factTable] = factSelectAvailable;
 $scope.availableTables.push(factTable);
 
@@ -91,20 +88,17 @@ KylinApp.controller('CubeDimensionsCtrl', function ($scope, 
$modal,MetaModel,cub
 var cols2 = $scope.getDimColumnsByTable(lookups[j].table);
 
 // Initialize selected available.
-var lookupAvailable = {};
 var lookupSelectAvailable = {};
 
 for (var k = 0; k < cols2.length; k++) {
 cols2[k].table = lookups[j].table;
 cols2[k].isLookup = true;
 
-lookupAvailable[cols2[k].name] = cols2[k];
-
 // Default not selected and not disabled.
 lookupSelectAvailable[cols2[k].name] = {selected: false, 
disabled: false};
 }
 
-$scope.availableColumns[lookups[j].table] = lookupAvailable;
+$scope.availableColumns[lookups[j].table] = cols2;
 $scope.selectedColumns[lookups[j].table] = lookupSelectAvailable;
 if($scope.availableTables.indexOf(lookups[j].table)==-1){
 $scope.availableTables.push(lookups[j].table);