[3/3] kylin git commit: KYLIN-1343 Upgrade to calcite 1.6 (with Edward Zhang)

2016-03-07 Thread liyang 
KYLIN-1343 Upgrade to calcite 1.6 (with Edward Zhang)


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f73abf6c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f73abf6c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f73abf6c

Branch: refs/heads/2.x-staging
Commit: f73abf6c2df97c1c89e7e6edf3865d82350f8472
Parents: 9021f17
Author: Yang Li 
Authored: Sun Mar 6 00:06:05 2016 +0800
Committer: Li Yang 
Committed: Tue Mar 8 15:54:05 2016 +0800

--
 .../adapter/enumerable/EnumerableJoin.java  |  145 -
 .../apache/calcite/runtime/SqlFunctions.java| 1317 ---
 .../apache/calcite/sql2rel/RelFieldTrimmer.java |   44 +
 .../calcite/sql2rel/SqlToRelConverter.java  | 8150 ++
 .../engine/mr/steps/MergeCuboidJobTest.java |1 -
 .../org/apache/kylin/jdbc/KylinConnection.java  |2 +-
 .../org/apache/kylin/jdbc/KylinJdbcFactory.java |5 +-
 .../java/org/apache/kylin/jdbc/KylinMeta.java   |   46 +-
 .../kylin/jdbc/KylinPreparedStatement.java  |2 +
 .../org/apache/kylin/jdbc/KylinResultSet.java   |5 +-
 kylin-it/src/test/resources/logging.properties  |2 +-
 .../src/test/resources/query/sql/query92.sql|   30 +
 .../resources/query/sql/query92.sql.disabled|   30 -
 .../src/test/resources/query/sql/query93.sql|   30 +
 .../resources/query/sql/query93.sql.disabled|   30 -
 .../src/test/resources/query/sql/query94.sql|   30 +
 .../resources/query/sql/query94.sql.disabled|   30 -
 .../src/test/resources/query/sql/query95.sql|   30 +
 .../resources/query/sql/query95.sql.disabled|   30 -
 pom.xml |2 +-
 .../kylin/query/optrule/OLAPJoinRule.java   |2 +-
 .../kylin/query/relnode/OLAPAggregateRel.java   |   10 +-
 .../kylin/query/relnode/OLAPFilterRel.java  |5 +-
 .../apache/kylin/query/relnode/OLAPJoinRel.java |   22 +-
 .../kylin/query/relnode/OLAPLimitRel.java   |5 +-
 .../kylin/query/relnode/OLAPProjectRel.java |5 +-
 .../apache/kylin/query/relnode/OLAPSortRel.java |5 +-
 .../kylin/query/relnode/OLAPTableScan.java  |9 +-
 .../relnode/OLAPToEnumerableConverter.java  |6 +-
 .../apache/kylin/query/schema/OLAPTable.java|   17 +-
 30 files changed, 4836 insertions(+), 5211 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/f73abf6c/atopcalcite/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableJoin.java
--
diff --git 
a/atopcalcite/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableJoin.java
 
b/atopcalcite/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableJoin.java
deleted file mode 100644
index a3c04f8..000
--- 
a/atopcalcite/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableJoin.java
+++ /dev/null
@@ -1,145 +0,0 @@
-/*
- * OVERRIDE POINT:
- * - constructor was private instead of protected 
- */
-
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.calcite.adapter.enumerable;
-
-import java.util.Set;
-
-import org.apache.calcite.linq4j.tree.BlockBuilder;
-import org.apache.calcite.linq4j.tree.Expression;
-import org.apache.calcite.linq4j.tree.Expressions;
-import org.apache.calcite.plan.RelOptCluster;
-import org.apache.calcite.plan.RelOptCost;
-import org.apache.calcite.plan.RelOptPlanner;
-import org.apache.calcite.plan.RelTraitSet;
-import org.apache.calcite.rel.InvalidRelException;
-import org.apache.calcite.rel.RelNode;
-import org.apache.calcite.rel.RelNodes;
-import org.apache.calcite.rel.core.EquiJoin;
-import org.apache.calcite.rel.core.JoinInfo;
-import org.apache.calcite.rel.core.JoinRelType;
-import org.apache.calcite.rel.metadata.RelMetadataQuery;
-import org.apache.calcite.rex.RexNode;
-import org.apache.calcite.util.BuiltInMethod;
-import org.apache.calcite.util.ImmutableIntList;
-import org.apache.calcite.util.Util;
-
-import com.google.common.collect.ImmutableList;
-
-/**

[1/3] kylin git commit: KYLIN-1343 Upgrade to calcite 1.6 (with Edward Zhang)

2016-03-07 Thread liyang 
Repository: kylin
Updated Branches:
  refs/heads/2.x-staging 9021f17d8 -> f73abf6c2


http://git-wip-us.apache.org/repos/asf/kylin/blob/f73abf6c/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MergeCuboidJobTest.java
--
diff --git 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MergeCuboidJobTest.java
 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MergeCuboidJobTest.java
index 7e530db..97b1ef2 100644
--- 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MergeCuboidJobTest.java
+++ 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MergeCuboidJobTest.java
@@ -44,7 +44,6 @@ public class MergeCuboidJobTest extends 
LocalFileMetadataTestCase {
 conf.set("fs.default.name", "file:///");
 conf.set("mapreduce.framework.name", "local");
 conf.set("mapreduce.application.framework.path", "");
-
 
 // for local runner out-of-memory issue
 conf.set("mapreduce.task.io.sort.mb", "10");

http://git-wip-us.apache.org/repos/asf/kylin/blob/f73abf6c/jdbc/src/main/java/org/apache/kylin/jdbc/KylinConnection.java
--
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinConnection.java 
b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinConnection.java
index 86c8d1d..6852998 100644
--- a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinConnection.java
+++ b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinConnection.java
@@ -118,7 +118,7 @@ public class KylinConnection extends AvaticaConnection {
 ArrayList columns = new ArrayList();
 Map internalParams = Collections. 
emptyMap();
 
-return new Meta.Signature(columns, sql, params, internalParams, 
CursorFactory.ARRAY);
+return new Meta.Signature(columns, sql, params, internalParams, 
CursorFactory.ARRAY, Meta.StatementType.SELECT);
 }
 
 private KylinJdbcFactory factory() {

http://git-wip-us.apache.org/repos/asf/kylin/blob/f73abf6c/jdbc/src/main/java/org/apache/kylin/jdbc/KylinJdbcFactory.java
--
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinJdbcFactory.java 
b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinJdbcFactory.java
index f1a4939..6aae983 100644
--- a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinJdbcFactory.java
+++ b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinJdbcFactory.java
@@ -33,6 +33,7 @@ import org.apache.calcite.avatica.AvaticaStatement;
 import org.apache.calcite.avatica.Meta.Frame;
 import org.apache.calcite.avatica.Meta.Signature;
 import org.apache.calcite.avatica.Meta.StatementHandle;
+import org.apache.calcite.avatica.QueryState;
 import org.apache.calcite.avatica.UnregisteredDriver;
 
 /**
@@ -93,9 +94,9 @@ public class KylinJdbcFactory implements AvaticaFactory {
 }
 
 @Override
-public AvaticaResultSet newResultSet(AvaticaStatement statement, Signature 
signature, TimeZone timeZone, Frame firstFrame) throws SQLException {
+public AvaticaResultSet newResultSet(AvaticaStatement statement, 
QueryState state, Signature signature, TimeZone timeZone, Frame firstFrame) 
throws SQLException {
 AvaticaResultSetMetaData resultSetMetaData = new 
AvaticaResultSetMetaData(statement, null, signature);
-return new KylinResultSet(statement, signature, resultSetMetaData, 
timeZone, firstFrame);
+return new KylinResultSet(statement, state, signature, 
resultSetMetaData, timeZone, firstFrame);
 }
 
 @Override

http://git-wip-us.apache.org/repos/asf/kylin/blob/f73abf6c/jdbc/src/main/java/org/apache/kylin/jdbc/KylinMeta.java
--
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinMeta.java 
b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinMeta.java
index cae39ad..05f7983 100644
--- a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinMeta.java
+++ b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinMeta.java
@@ -30,6 +30,10 @@ import java.util.regex.Pattern;
 import org.apache.calcite.avatica.AvaticaUtils;
 import org.apache.calcite.avatica.ColumnMetaData;
 import org.apache.calcite.avatica.MetaImpl;
+import org.apache.calcite.avatica.MissingResultsException;
+import org.apache.calcite.avatica.NoSuchStatementException;
+import org.apache.calcite.avatica.QueryState;
+import org.apache.calcite.avatica.remote.TypedValue;
 
 import com.google.common.collect.ImmutableList;
 
@@ -55,6 +59,13 @@ public class KylinMeta extends MetaImpl {
 result.signature = connection().mockPreparedSignature(sql);
 return result;
 }
+
+// real execution happens in KylinResultSet.execute()
+@Override
+public ExecuteResult execute(StatementHandle sh, List 
parameterValues, long maxRowCount) throws NoSuchStatementException {
+final MetaResultSet metaResultSet =

[2/3] kylin git commit: KYLIN-1343 Upgrade to calcite 1.6 (with Edward Zhang)

2016-03-07 Thread liyang 
http://git-wip-us.apache.org/repos/asf/kylin/blob/f73abf6c/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
--
diff --git 
a/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java 
b/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
index c184f0b..aed7c27 100644
--- 
a/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
+++ 
b/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
@@ -18,6 +18,7 @@
 
 package org.apache.calcite.sql2rel;
 
+import org.apache.calcite.avatica.util.Spaces;
 import org.apache.calcite.linq4j.Ord;
 import org.apache.calcite.plan.Convention;
 import org.apache.calcite.plan.RelOptCluster;
@@ -25,26 +26,32 @@ import org.apache.calcite.plan.RelOptPlanner;
 import org.apache.calcite.plan.RelOptSamplingParameters;
 import org.apache.calcite.plan.RelOptTable;
 import org.apache.calcite.plan.RelOptUtil;
+import org.apache.calcite.plan.RelTraitSet;
 import org.apache.calcite.prepare.Prepare;
 import org.apache.calcite.prepare.RelOptTableImpl;
 import org.apache.calcite.rel.RelCollation;
+import org.apache.calcite.rel.RelCollationTraitDef;
 import org.apache.calcite.rel.RelCollations;
 import org.apache.calcite.rel.RelFieldCollation;
 import org.apache.calcite.rel.RelNode;
-import org.apache.calcite.rel.RelShuttle;
+import org.apache.calcite.rel.RelRoot;
+import org.apache.calcite.rel.SingleRel;
 import org.apache.calcite.rel.core.Aggregate;
 import org.apache.calcite.rel.core.AggregateCall;
 import org.apache.calcite.rel.core.Collect;
 import org.apache.calcite.rel.core.CorrelationId;
+import org.apache.calcite.rel.core.Filter;
 import org.apache.calcite.rel.core.Join;
 import org.apache.calcite.rel.core.JoinInfo;
 import org.apache.calcite.rel.core.JoinRelType;
 import org.apache.calcite.rel.core.Project;
 import org.apache.calcite.rel.core.RelFactories;
 import org.apache.calcite.rel.core.Sample;
+import org.apache.calcite.rel.core.Sort;
 import org.apache.calcite.rel.core.Uncollect;
 import org.apache.calcite.rel.logical.LogicalAggregate;
 import org.apache.calcite.rel.logical.LogicalCorrelate;
+import org.apache.calcite.rel.logical.LogicalFilter;
 import org.apache.calcite.rel.logical.LogicalIntersect;
 import org.apache.calcite.rel.logical.LogicalJoin;
 import org.apache.calcite.rel.logical.LogicalMinus;
@@ -56,6 +63,7 @@ import org.apache.calcite.rel.logical.LogicalTableScan;
 import org.apache.calcite.rel.logical.LogicalUnion;
 import org.apache.calcite.rel.logical.LogicalValues;
 import org.apache.calcite.rel.metadata.RelColumnMapping;
+import org.apache.calcite.rel.stream.Delta;
 import org.apache.calcite.rel.stream.LogicalDelta;
 import org.apache.calcite.rel.type.RelDataType;
 import org.apache.calcite.rel.type.RelDataTypeFactory;
@@ -72,8 +80,8 @@ import org.apache.calcite.rex.RexLiteral;
 import org.apache.calcite.rex.RexNode;
 import org.apache.calcite.rex.RexRangeRef;
 import org.apache.calcite.rex.RexShuttle;
+import org.apache.calcite.rex.RexSubQuery;
 import org.apache.calcite.rex.RexUtil;
-import org.apache.calcite.rex.RexVisitorImpl;
 import org.apache.calcite.rex.RexWindowBound;
 import org.apache.calcite.schema.ModifiableTable;
 import org.apache.calcite.schema.ModifiableView;
@@ -85,6 +93,7 @@ import org.apache.calcite.sql.SemiJoinType;
 import org.apache.calcite.sql.SqlAggFunction;
 import org.apache.calcite.sql.SqlBasicCall;
 import org.apache.calcite.sql.SqlCall;
+import org.apache.calcite.sql.SqlCallBinding;
 import org.apache.calcite.sql.SqlDataTypeSpec;
 import org.apache.calcite.sql.SqlDelete;
 import org.apache.calcite.sql.SqlDynamicParam;
@@ -102,10 +111,12 @@ import org.apache.calcite.sql.SqlNodeList;
 import org.apache.calcite.sql.SqlNumericLiteral;
 import org.apache.calcite.sql.SqlOperator;
 import org.apache.calcite.sql.SqlOperatorTable;
+import org.apache.calcite.sql.SqlOrderBy;
 import org.apache.calcite.sql.SqlSampleSpec;
 import org.apache.calcite.sql.SqlSelect;
 import org.apache.calcite.sql.SqlSelectKeyword;
 import org.apache.calcite.sql.SqlSetOperator;
+import org.apache.calcite.sql.SqlUnnestOperator;
 import org.apache.calcite.sql.SqlUpdate;
 import org.apache.calcite.sql.SqlUtil;
 import org.apache.calcite.sql.SqlValuesOperator;
@@ -138,8 +149,10 @@ import org.apache.calcite.sql.validate.SqlValidatorImpl;
 import org.apache.calcite.sql.validate.SqlValidatorNamespace;
 import org.apache.calcite.sql.validate.SqlValidatorScope;
 import org.apache.calcite.sql.validate.SqlValidatorUtil;
+import org.apache.calcite.tools.RelBuilder;
 import org.apache.calcite.util.ImmutableBitSet;
 import org.apache.calcite.util.ImmutableIntList;
+import org.apache.calcite.util.Litmus;
 import org.apache.calcite.util.NlsString;
 import org.apache.calcite.util.NumberUtil;
 import org.apache.calcite.util.Pair;
@@ -158,15 +171,16 @@ import com.google.common.collect.Sets;
 import

[1/2] kylin git commit: KYLIN-1474 expose list, remove and cat in metastore.sh

2016-03-07 Thread mahongbin 
Repository: kylin
Updated Branches:
  refs/heads/2.0-rc 85e9c30d7 -> 1fed301db


KYLIN-1474 expose list, remove and cat in metastore.sh

Signed-off-by: Hongbin Ma 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/278893cd
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/278893cd
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/278893cd

Branch: refs/heads/2.0-rc
Commit: 278893cd66e2a0325325880e027a63d927cfc4b3
Parents: 85e9c30
Author: John Zhao 
Authored: Mon Mar 7 14:05:59 2016 -0800
Committer: Hongbin Ma 
Committed: Tue Mar 8 15:35:47 2016 +0800

--
 build/bin/metastore.sh  | 24 
 .../kylin/common/persistence/ResourceTool.java  |  3 +++
 2 files changed, 27 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/278893cd/build/bin/metastore.sh
--
diff --git a/build/bin/metastore.sh b/build/bin/metastore.sh
index 05041f9..8908351 100755
--- a/build/bin/metastore.sh
+++ b/build/bin/metastore.sh
@@ -47,6 +47,27 @@ then
 echo "Starting restoring $_file"
 ${KYLIN_HOME}/bin/kylin.sh 
org.apache.kylin.common.persistence.ResourceTool upload $_file
 
+elif [ "$1" == "list" ]
+then
+
+_file=$2
+echo "Starting list $_file"
+${KYLIN_HOME}/bin/kylin.sh 
org.apache.kylin.common.persistence.ResourceTool list $_file
+
+elif [ "$1" == "remove" ]
+then
+
+_file=$2
+echo "Starting remove $_file"
+${KYLIN_HOME}/bin/kylin.sh 
org.apache.kylin.common.persistence.ResourceTool remove $_file
+
+elif [ "$1" == "cat" ]
+then
+
+_file=$2
+echo "Starting cat $_file"
+${KYLIN_HOME}/bin/kylin.sh 
org.apache.kylin.common.persistence.ResourceTool cat $_file
+
 elif [ "$1" == "reset" ]
 then
 
@@ -61,6 +82,9 @@ else
 echo "usage: metastore.sh backup"
 echo "   metastore.sh reset"
 echo "   metastore.sh restore PATH_TO_LOCAL_META"
+echo "   metastore.sh list RESOURCE_PATH"
+echo "   metastore.sh cat RESOURCE_PATH"
+echo "   metastore.sh remove RESOURCE_PATH"
 echo "   metastore.sh clean [--delete true]"
 exit 1
 fi

http://git-wip-us.apache.org/repos/asf/kylin/blob/278893cd/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
 
b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
index 8faeadf..0975d57 100644
--- 
a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
+++ 
b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
@@ -39,6 +39,9 @@ public class ResourceTool {
 System.out.println("Usage: ResourceTool list  RESOURCE_PATH");
 System.out.println("Usage: ResourceTool download  LOCAL_DIR");
 System.out.println("Usage: ResourceTool uploadLOCAL_DIR");
+System.out.println("Usage: ResourceTool reset");
+System.out.println("Usage: ResourceTool remove RESOURCE_PATH");
+System.out.println("Usage: ResourceTool cat RESOURCE_PATH");
 return;
 }

[1/2] kylin git commit: KYLIN-1475 Inject ehcache manager for any test case that will touch ehcache manager

2016-03-07 Thread mahongbin 
Repository: kylin
Updated Branches:
  refs/heads/2.0-rc e177ef50f -> 85e9c30d7


KYLIN-1475 Inject ehcache manager for any test case that will touch ehcache 
manager


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/83f49c34
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/83f49c34
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/83f49c34

Branch: refs/heads/2.0-rc
Commit: 83f49c34f0302bfcc323a760bf6097051006546f
Parents: e177ef5
Author: Hongbin Ma 
Authored: Tue Mar 8 14:30:56 2016 +0800
Committer: Hongbin Ma 
Committed: Tue Mar 8 14:30:56 2016 +0800

--
 .../dict/TupleFilterDictionaryTranslater.java   | 166 ---
 .../dict/TupleFilterFunctionTransformer.java| 161 ++
 .../filter/ITupleFilterTransformer.java |  23 +++
 .../metadata/filter/ITupleFilterTranslator.java |  26 ---
 .../metadata/filter/function/BuiltInMethod.java |  24 ++-
 .../cache/AbstractCacheFledgedQuery.java|  29 +---
 .../kylin/storage/cache/DynamicCacheTest.java   |  31 +++-
 .../kylin/storage/cache/StaticCacheTest.java|  32 +++-
 .../apache/kylin/query/ITKylinQueryTest.java|  15 ++
 .../kylin/storage/hbase/ITStorageTest.java  |  22 ++-
 .../common/coprocessor/FilterDecorator.java |   6 +-
 .../hbase/cube/v2/CubeSegmentScanner.java   |   6 +-
 12 files changed, 285 insertions(+), 256 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/83f49c34/core-dictionary/src/main/java/org/apache/kylin/dict/TupleFilterDictionaryTranslater.java
--
diff --git 
a/core-dictionary/src/main/java/org/apache/kylin/dict/TupleFilterDictionaryTranslater.java
 
b/core-dictionary/src/main/java/org/apache/kylin/dict/TupleFilterDictionaryTranslater.java
deleted file mode 100644
index fec6a85..000
--- 
a/core-dictionary/src/main/java/org/apache/kylin/dict/TupleFilterDictionaryTranslater.java
+++ /dev/null
@@ -1,166 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.dict;
-
-import com.google.common.primitives.Primitives;
-import org.apache.kylin.dict.Dictionary;
-import org.apache.kylin.dict.IDictionaryAware;
-import org.apache.kylin.metadata.filter.ColumnTupleFilter;
-import org.apache.kylin.metadata.filter.CompareTupleFilter;
-import org.apache.kylin.metadata.filter.ConstantTupleFilter;
-import org.apache.kylin.metadata.filter.FunctionTupleFilter;
-import org.apache.kylin.metadata.filter.ITupleFilterTranslator;
-import org.apache.kylin.metadata.filter.LogicalTupleFilter;
-import org.apache.kylin.metadata.filter.TupleFilter;
-import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ListIterator;
-
-/**
- * Created by dongli on 1/7/16.
- */
-public class TupleFilterDictionaryTranslater implements ITupleFilterTranslator 
{
-public static final Logger logger = 
LoggerFactory.getLogger(TupleFilterDictionaryTranslater.class);
-
-private IDictionaryAware dictionaryAware;
-
-public TupleFilterDictionaryTranslater(IDictionaryAware dictionaryAware) {
-this.dictionaryAware = dictionaryAware;
-}
-
-@Override
-public TupleFilter translate(TupleFilter tupleFilter) {
-TupleFilter translated = null;
-if (tupleFilter instanceof CompareTupleFilter) {
-translated = translateCompareTupleFilter((CompareTupleFilter) 
tupleFilter);
-if (translated != null) {
-logger.info("Translated {" + tupleFilter + "} to IN clause: {" 
+ translated + "}");
-}
-} else if (tupleFilter instanceof FunctionTupleFilter) {
-translated = translateFunctionTupleFilter((FunctionTupleFilter) 
tupleFilter);
-if (translated != null) {
-logger.info("Translated {" + tupleFilter + "} to IN clause: {" 
+ translated + "}");

[2/2] kylin git commit: KYLIN-1471 - LIMIT after having clause should not be pushed down to storage context

2016-03-07 Thread mahongbin 
KYLIN-1471 - LIMIT after having clause should not be pushed down to storage 
context


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/85e9c30d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/85e9c30d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/85e9c30d

Branch: refs/heads/2.0-rc
Commit: 85e9c30d797693b4d9246b5e84888fb2a3f5ddc6
Parents: 83f49c3
Author: Hongbin Ma 
Authored: Fri Mar 4 16:00:11 2016 +0800
Committer: Hongbin Ma 
Committed: Tue Mar 8 14:31:16 2016 +0800

--
 .../resources/query/sql_tableau/query29.sql | 29 
 .../apache/kylin/query/relnode/OLAPContext.java |  1 +
 .../kylin/query/relnode/OLAPFilterRel.java  |  2 ++
 .../kylin/query/relnode/OLAPLimitRel.java   | 21 --
 4 files changed, 44 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/85e9c30d/kylin-it/src/test/resources/query/sql_tableau/query29.sql
--
diff --git a/kylin-it/src/test/resources/query/sql_tableau/query29.sql 
b/kylin-it/src/test/resources/query/sql_tableau/query29.sql
new file mode 100644
index 000..0858087
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_tableau/query29.sql
@@ -0,0 +1,29 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+SELECT * 
+ FROM ( 
+ select test_kylin_fact.lstg_format_name, 
test_cal_dt.week_beg_dt,sum(test_kylin_fact.price) as GMV 
+ , count(*) as TRANS_CNT 
+ from test_kylin_fact 
+ inner JOIN edw.test_cal_dt as test_cal_dt 
+ ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt 
+ where test_cal_dt.week_beg_dt between DATE '2013-05-01' and DATE '2013-08-01' 
+ group by test_kylin_fact.lstg_format_name, test_cal_dt.week_beg_dt 
+ ) "TableauSQL" 
+ LIMIT 1 

http://git-wip-us.apache.org/repos/asf/kylin/blob/85e9c30d/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java
--
diff --git 
a/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java 
b/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java
index 6865457..a6844e1 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java
@@ -102,6 +102,7 @@ public class OLAPContext {
 public OLAPTableScan firstTableScan = null; // to be fact table scan 
except "select * from lookupTable"
 public TupleInfo returnTupleInfo = null;
 public boolean afterAggregate = false;
+public boolean afterSkippedFilter = false;
 public boolean afterJoin = false;
 public boolean hasJoin = false;
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/85e9c30d/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
--
diff --git 
a/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java 
b/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
index 7b8bfdb..a847890 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
@@ -271,6 +271,8 @@ public class OLAPFilterRel extends Filter implements 
OLAPRel {
 // only translate where clause and don't translate having clause
 if (!context.afterAggregate) {
 translateFilter(context);
+} else {
+context.afterSkippedFilter = true;//having clause is skipped
 }
 }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/85e9c30d/query/src/main/java/org/apache/kylin/query/relnode/OLAPLimitRel.java
--
diff --git 
a/query/src/main/java/org/apache/kylin/query/relnode/OLAPLimitRel.java 
b/query/src/main/java/org/apache/kylin/query/relnode/OLAPLimitRel.java
index 572a5c7..82aa9de 100644
---

kylin git commit: initial commit for KYLIN-1431

2016-03-07 Thread shaofengshi 
Repository: kylin
Updated Branches:
  refs/heads/2.x-staging 0ec3ed0e8 -> 9021f17d8


initial commit for KYLIN-1431


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9021f17d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9021f17d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9021f17d

Branch: refs/heads/2.x-staging
Commit: 9021f17d85be01bf34b48a7a31be82f53ceb9c8f
Parents: 0ec3ed0
Author: shaofengshi 
Authored: Wed Mar 2 11:16:46 2016 +0800
Committer: shaofengshi 
Committed: Tue Mar 8 15:07:04 2016 +0800

--
 .../org/apache/kylin/job/CubeMetaExtractor.java |   2 +-
 build/bin/streaming_build.sh|   4 +-
 build/bin/streaming_fillgap.sh  |   5 +-
 .../kylin/metadata/model/ISourceAware.java  |   1 +
 .../kylin/engine/streaming/BootstrapConfig.java |  20 +-
 .../kylin/engine/streaming/IStreamingInput.java |   3 +-
 .../streaming/OneOffStreamingBuilder.java   |  17 +-
 .../kylin/engine/streaming/StreamingConfig.java |  33 +--
 .../engine/streaming/StreamingManager.java  |  12 +
 .../engine/streaming/cli/StreamingCLI.java  |  21 +-
 .../engine/streaming/util/StreamingUtils.java   |  18 +-
 .../kafka/default.streaming_table.json  |  21 ++
 .../localmeta/kafka/kafka_test.json |  20 --
 .../kafka/test_streaming_table_cube.json|  22 --
 .../kafka/test_streaming_table_ii.json  |  22 --
 .../streaming/default.streaming_table.json  |   6 +
 .../localmeta/streaming/kafka_test.json |  20 --
 .../streaming/test_streaming_table_cube.json|   8 -
 .../streaming/test_streaming_table_ii.json  |   8 -
 .../kylin/provision/BuildCubeWithStream.java|  16 +-
 .../kylin/rest/controller/CubeController.java   | 234 ---
 .../rest/controller/StreamingController.java|   4 +-
 .../kylin/rest/service/StreamingService.java|  18 +-
 .../kylin/source/kafka/KafkaStreamingInput.java |  78 ---
 .../kylin/source/kafka/StreamingParser.java |   6 +-
 25 files changed, 163 insertions(+), 456 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/9021f17d/assembly/src/test/java/org/apache/kylin/job/CubeMetaExtractor.java
--
diff --git a/assembly/src/test/java/org/apache/kylin/job/CubeMetaExtractor.java 
b/assembly/src/test/java/org/apache/kylin/job/CubeMetaExtractor.java
index 527ef0a..ef27ade 100644
--- a/assembly/src/test/java/org/apache/kylin/job/CubeMetaExtractor.java
+++ b/assembly/src/test/java/org/apache/kylin/job/CubeMetaExtractor.java
@@ -227,7 +227,7 @@ public class CubeMetaExtractor extends AbstractApplication {
 
 private void dealWithStreaming(CubeInstance cube) {
 for (StreamingConfig streamingConfig : 
streamingManager.listAllStreaming()) {
-if (streamingConfig.getCubeName() != null && 
streamingConfig.getCubeName().equalsIgnoreCase(cube.getName())) {
+if (streamingConfig.getName() != null && 
streamingConfig.getName().equalsIgnoreCase(cube.getFactTable())) {
 
requiredResources.add(StreamingConfig.concatResourcePath(streamingConfig.getName()));
 
requiredResources.add(KafkaConfig.concatResourcePath(streamingConfig.getName()));
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/9021f17d/build/bin/streaming_build.sh
--
diff --git a/build/bin/streaming_build.sh b/build/bin/streaming_build.sh
index a96ecc1..ed19036 100644
--- a/build/bin/streaming_build.sh
+++ b/build/bin/streaming_build.sh
@@ -20,7 +20,7 @@
 source /etc/profile
 source ~/.bash_profile
 
-STREAMING=$1
+CUBE=$1
 INTERVAL=$2
 DELAY=$3
 CURRENT_TIME_IN_SECOND=`date +%s`
@@ -30,4 +30,4 @@ END=$(($CURRENT_TIME - CURRENT_TIME%INTERVAL - DELAY + 
INTERVAL))
 
 ID="$START"_"$END"
 echo "building for ${ID}" >> ${KYLIN_HOME}/logs/build_trace.log
-sh ${KYLIN_HOME}/bin/kylin.sh streaming start ${STREAMING} ${ID} -start 
${START} -end ${END} -streaming ${STREAMING}
\ No newline at end of file
+sh ${KYLIN_HOME}/bin/kylin.sh streaming start ${CUBE} ${ID} -start ${START} 
-end ${END} -cube ${CUBE}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/kylin/blob/9021f17d/build/bin/streaming_fillgap.sh
--
diff --git a/build/bin/streaming_fillgap.sh b/build/bin/streaming_fillgap.sh
index 74d9037..c67809a 100644
--- a/build/bin/streaming_fillgap.sh
+++ b/build/bin/streaming_fillgap.sh
@@ -20,8 +20,7 @@
 source /etc/profile
 source ~/.bash_profile
 
-streaming=$1
-margin=$2
+cube=$1
 
 cd ${KYLIN_HOME}
-sh ${KYLIN_HOME}/bin/kylin.sh streaming start ${streaming} fillgap

[7/8] kylin git commit: KYLIN-1417 Change to use TreeMap to allow null as value

2016-03-07 Thread shaofengshi 
KYLIN-1417 Change to use TreeMap to allow null as value


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d1a574b8
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d1a574b8
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d1a574b8

Branch: refs/heads/2.x-staging
Commit: d1a574b80ef35ebdae69718998a2dfcaeafc3cbc
Parents: 50aab0b
Author: shaofengshi 
Authored: Tue Feb 16 10:29:47 2016 +0800
Committer: shaofengshi 
Committed: Tue Mar 8 12:21:40 2016 +0800

--
 .../apache/kylin/source/kafka/TimedJsonStreamParser.java  | 10 --
 1 file changed, 4 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/d1a574b8/source-kafka/src/main/java/org/apache/kylin/source/kafka/TimedJsonStreamParser.java
--
diff --git 
a/source-kafka/src/main/java/org/apache/kylin/source/kafka/TimedJsonStreamParser.java
 
b/source-kafka/src/main/java/org/apache/kylin/source/kafka/TimedJsonStreamParser.java
index 0907623..e3075d5 100644
--- 
a/source-kafka/src/main/java/org/apache/kylin/source/kafka/TimedJsonStreamParser.java
+++ 
b/source-kafka/src/main/java/org/apache/kylin/source/kafka/TimedJsonStreamParser.java
@@ -35,11 +35,7 @@
 package org.apache.kylin.source.kafka;
 
 import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
 
 import kafka.message.MessageAndOffset;
 
@@ -102,7 +98,9 @@ public final class TimedJsonStreamParser extends 
StreamingParser {
 @Override
 public StreamingMessage parse(MessageAndOffset messageAndOffset) {
 try {
-Map root = mapper.readValue(new 
ByteBufferBackedInputStream(messageAndOffset.message().payload()), mapType);
+Map message = mapper.readValue(new 
ByteBufferBackedInputStream(messageAndOffset.message().payload()), mapType);
+Map root = new TreeMap(String.CASE_INSENSITIVE_ORDER);
+root.putAll(message);
 String tsStr = root.get(tsColName);
 //Preconditions.checkArgument(!StringUtils.isEmpty(tsStr), 
"Timestamp field " + tsColName + //
 //" cannot be null, the message offset is " + 
messageAndOffset.getOffset() + " content is " + new 
String(messageAndOffset.getRawData()));

[6/8] kylin git commit: KYLIN-1421 Cube "source record" is always zero for streaming

2016-03-07 Thread shaofengshi 
KYLIN-1421 Cube "source record" is always zero for streaming


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/50aab0b0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/50aab0b0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/50aab0b0

Branch: refs/heads/2.x-staging
Commit: 50aab0b08ea1532998db1f11084c1ae908a42e56
Parents: 9352e5a
Author: shaofengshi 
Authored: Tue Feb 16 10:30:05 2016 +0800
Committer: shaofengshi 
Committed: Tue Mar 8 12:20:05 2016 +0800

--
 .../apache/kylin/engine/streaming/cube/StreamingCubeBuilder.java| 1 +
 1 file changed, 1 insertion(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/50aab0b0/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/cube/StreamingCubeBuilder.java
--
diff --git 
a/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/cube/StreamingCubeBuilder.java
 
b/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/cube/StreamingCubeBuilder.java
index ec2ad91..b4182fe 100644
--- 
a/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/cube/StreamingCubeBuilder.java
+++ 
b/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/cube/StreamingCubeBuilder.java
@@ -117,6 +117,7 @@ public class StreamingCubeBuilder implements 
StreamingBatchBuilder {
 try {
 CubeSegment segment = cubeManager.appendSegments(cubeInstance, 
streamingBatch.getTimeRange().getFirst(), 
streamingBatch.getTimeRange().getSecond(), false, false);
 segment.setLastBuildJobID(segment.getUuid()); // give a fake job id
+segment.setInputRecords(streamingBatch.getMessages().size());
 return segment;
 } catch (IOException e) {
 throw new RuntimeException("failed to create IBuildable", e);

[2/8] kylin git commit: KYLIN-1420 Query returns empty result on partition column's boundary condition

2016-03-07 Thread shaofengshi 
KYLIN-1420 Query returns empty result on partition column's boundary condition


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3736f72c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3736f72c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3736f72c

Branch: refs/heads/2.x-staging
Commit: 3736f72cc756af28a39a473efc47bec943ba7fc9
Parents: 3af7d4a
Author: shaofengshi 
Authored: Mon Feb 15 15:52:15 2016 +0800
Committer: shaofengshi 
Committed: Tue Mar 8 12:14:18 2016 +0800

--
 .../main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/3736f72c/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java 
b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java
index a72426d..d314dde 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java
@@ -108,8 +108,8 @@ public class GTScanRangePlanner {
 for (ColumnRange range : andDimRanges) {
 if (partitionColRef != null && 
range.column.equals(partitionColRef)) {
 if 
(rangeStartEndComparator.comparator.compare(segmentStartAndEnd.getFirst(), 
range.end) <= 0 //
-&& 
rangeStartEndComparator.comparator.compare(range.begin, 
segmentStartAndEnd.getSecond()) < 0) {
-//segment range is [Closed,Open)
+&& 
rangeStartEndComparator.comparator.compare(range.begin, 
segmentStartAndEnd.getSecond()) <= 0) {
+//segment range is [Closed,Open), but 
segmentStartAndEnd.getSecond() might be rounded, so use <=. 
 } else {
 logger.debug("Pre-check partition col filter failed, 
partitionColRef {}, segment start {}, segment end {}, range begin {}, range end 
{}",//
 new Object[] { partitionColRef, 
makeReadable(segmentStartAndEnd.getFirst()), 
makeReadable(segmentStartAndEnd.getSecond()), makeReadable(range.begin), 
makeReadable(range.end) });

[4/8] kylin git commit: KYLIN-1387 Streaming cubing doesn't generate cuboids files on HDFS, cause cube merge failure

2016-03-07 Thread shaofengshi 
KYLIN-1387 Streaming cubing doesn't generate cuboids files on HDFS, cause cube 
merge failure

Conflicts:

engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MapContextGTRecordWriter.java


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/929c7a49
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/929c7a49
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/929c7a49

Branch: refs/heads/2.x-staging
Commit: 929c7a4908a3cd655ab31e71eb6453971f3acd36
Parents: 250978d
Author: shaofengshi 
Authored: Tue Feb 2 17:34:46 2016 +0800
Committer: shaofengshi 
Committed: Tue Mar 8 12:19:36 2016 +0800

--
 .../cube/inmemcubing/CompoundCuboidWriter.java  | 57 ++
 .../kylin/cube/inmemcubing/ICuboidWriter.java   |  4 +-
 .../kylin/job/constant/ExecutableConstants.java |  1 +
 .../kylin/engine/mr/steps/KVGTRecordWriter.java | 81 
 .../mr/steps/MapContextGTRecordWriter.java  | 68 ++--
 .../streaming/cube/StreamingCubeBuilder.java| 12 ++-
 .../storage/hbase/steps/HBaseCuboidWriter.java  | 24 +++---
 .../hbase/steps/HBaseMROutput2Transition.java   |  2 +-
 .../kylin/storage/hbase/steps/HBaseMRSteps.java |  2 +-
 .../hbase/steps/HBaseStreamingOutput.java   |  8 +-
 .../hbase/steps/SequenceFileCuboidWriter.java   | 75 ++
 11 files changed, 254 insertions(+), 80 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/929c7a49/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/CompoundCuboidWriter.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/CompoundCuboidWriter.java
 
b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/CompoundCuboidWriter.java
new file mode 100644
index 000..46eef50
--- /dev/null
+++ 
b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/CompoundCuboidWriter.java
@@ -0,0 +1,57 @@
+/*
+ *  Licensed to the Apache Software Foundation (ASF) under one or more
+ *  contributor license agreements. See the NOTICE file distributed with
+ *  this work for additional information regarding copyright ownership.
+ *  The ASF licenses this file to You under the Apache License, Version 2.0
+ *  (the "License"); you may not use this file except in compliance with
+ *  the License. You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.kylin.cube.inmemcubing;
+
+import org.apache.kylin.gridtable.GTRecord;
+
+import java.io.IOException;
+
+/**
+ */
+public class CompoundCuboidWriter implements ICuboidWriter {
+
+private Iterable cuboidWriters;
+
+public CompoundCuboidWriter(Iterable cuboidWriters) {
+this.cuboidWriters = cuboidWriters;
+
+}
+
+@Override
+public void write(long cuboidId, GTRecord record) throws IOException {
+for (ICuboidWriter writer : cuboidWriters) {
+writer.write(cuboidId, record);
+}
+}
+
+@Override
+public void flush() throws IOException {
+for (ICuboidWriter writer : cuboidWriters) {
+writer.flush();
+}
+
+}
+
+@Override
+public void close() throws IOException {
+for (ICuboidWriter writer : cuboidWriters) {
+writer.close();
+}
+
+}
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/929c7a49/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/ICuboidWriter.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/ICuboidWriter.java 
b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/ICuboidWriter.java
index 9e26e5e..e6cfa02 100644
--- 
a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/ICuboidWriter.java
+++ 
b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/ICuboidWriter.java
@@ -27,7 +27,7 @@ public interface ICuboidWriter {
 
 void write(long cuboidId, GTRecord record) throws IOException;
 
-void flush();
+void flush() throws IOException;
 
-void close();
+void close() throws IOException;
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/929c7a49/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java
--
diff --git

[2/2] kylin git commit: KYLIN-1466 Update kylin.sh and bring denpendencies to runnable classes

2016-03-07 Thread lidong 
KYLIN-1466 Update kylin.sh and bring denpendencies to runnable classes


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e177ef50
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e177ef50
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e177ef50

Branch: refs/heads/2.0-rc
Commit: e177ef50ffbf9f7e10470e1241ebe44f553d9007
Parents: 081bf87
Author: lidongsjtu 
Authored: Thu Mar 3 23:26:32 2016 +0800
Committer: lidongsjtu 
Committed: Tue Mar 8 12:09:10 2016 +0800

--
 build/bin/kylin.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/e177ef50/build/bin/kylin.sh
--
diff --git a/build/bin/kylin.sh b/build/bin/kylin.sh
index f52f923..cb83015 100644
--- a/build/bin/kylin.sh
+++ b/build/bin/kylin.sh
@@ -201,7 +201,7 @@ then
 
 export 
HBASE_CLASSPATH=${KYLIN_HOME}/lib/*:$hive_dependency:${HBASE_CLASSPATH}
 
-exec hbase -Dlog4j.configuration=kylin-log4j.properties "$@"
+exec hbase ${KYLIN_EXTRA_START_OPTS} 
-Dkylin.hive.dependency=${hive_dependency} 
-Dkylin.hbase.dependency=${hbase_dependency} 
-Dlog4j.configuration=kylin-log4j.properties "$@"
 
 else
 echo "usage: kylin.sh start or kylin.sh stop"

[1/2] kylin git commit: KYLIN-1445 Check HIVE_CONF directory before startup Kylin instance

2016-03-07 Thread lidong 
Repository: kylin
Updated Branches:
  refs/heads/2.0-rc aa86b31ef -> e177ef50f


KYLIN-1445 Check HIVE_CONF directory before startup Kylin instance


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/081bf87d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/081bf87d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/081bf87d

Branch: refs/heads/2.0-rc
Commit: 081bf87dcdf9eef91f0594b34335abbef94baad7
Parents: aa86b31
Author: lidongsjtu 
Authored: Wed Feb 24 15:42:52 2016 +0800
Committer: lidongsjtu 
Committed: Tue Mar 8 12:08:23 2016 +0800

--
 build/bin/find-hive-dependency.sh | 24 +---
 1 file changed, 21 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/081bf87d/build/bin/find-hive-dependency.sh
--
diff --git a/build/bin/find-hive-dependency.sh 
b/build/bin/find-hive-dependency.sh
index 47f0fbc..55ee047 100644
--- a/build/bin/find-hive-dependency.sh
+++ b/build/bin/find-hive-dependency.sh
@@ -23,6 +23,13 @@ hive_classpath=`echo $hive_env | grep 'env:CLASSPATH' | awk 
-F '=' '{print $2}'`
 arr=(`echo $hive_classpath | cut -d ":"  --output-delimiter=" " -f 1-`)
 hive_conf_path=
 hive_exec_path=
+
+if [ -n "$HIVE_CONF" ]
+then
+echo "HIVE_CONF is set to: $HIVE_CONF, use it to locate hive 
configurations."
+hive_conf_path=$HIVE_CONF
+fi
+
 for data in ${arr[@]}
 do
 result=`echo $data | grep -e 'hive-exec[a-z0-9A-Z\.-]*jar'`
@@ -30,13 +37,24 @@ do
 then
 hive_exec_path=$data
 fi
-result=`echo $data | grep -e 'hive[^/]*/conf'`
-if [ $result ]
+
+# in some versions of hive config is not in hive's classpath, find it 
separately
+if [ -z "$hive_conf_path" ]
 then
-hive_conf_path=$data
+result=`echo $data | grep -e 'hive[^/]*/conf'`
+if [ $result ]
+then
+hive_conf_path=$data
+fi
 fi
 done
 
+if [ -z "$hive_conf_path" ]
+then
+echo "Couldn't find hive configuration directory. Please set HIVE_CONF to 
the path which contains hive-site.xml."
+exit 1
+fi
+
 # in some versions of hive hcatalog is not in hive's classpath, find it 
separately
 if [ -z "$HCAT_HOME" ]
 then

kylin git commit: KYLIN-1474 expose list, remove and cat in metastore.sh

2016-03-07 Thread mahongbin 
Repository: kylin
Updated Branches:
  refs/heads/2.x-staging 11dc04b61 -> 17c33dc49


KYLIN-1474 expose list, remove and cat in metastore.sh

Signed-off-by: Hongbin Ma 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/17c33dc4
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/17c33dc4
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/17c33dc4

Branch: refs/heads/2.x-staging
Commit: 17c33dc4929121cdb48c3904dc337cd456af845d
Parents: 11dc04b
Author: John Zhao 
Authored: Mon Mar 7 14:05:59 2016 -0800
Committer: Hongbin Ma 
Committed: Tue Mar 8 10:31:12 2016 +0800

--
 build/bin/metastore.sh  | 24 
 .../kylin/common/persistence/ResourceTool.java  |  3 +++
 2 files changed, 27 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/17c33dc4/build/bin/metastore.sh
--
diff --git a/build/bin/metastore.sh b/build/bin/metastore.sh
index 05041f9..8908351 100755
--- a/build/bin/metastore.sh
+++ b/build/bin/metastore.sh
@@ -47,6 +47,27 @@ then
 echo "Starting restoring $_file"
 ${KYLIN_HOME}/bin/kylin.sh 
org.apache.kylin.common.persistence.ResourceTool upload $_file
 
+elif [ "$1" == "list" ]
+then
+
+_file=$2
+echo "Starting list $_file"
+${KYLIN_HOME}/bin/kylin.sh 
org.apache.kylin.common.persistence.ResourceTool list $_file
+
+elif [ "$1" == "remove" ]
+then
+
+_file=$2
+echo "Starting remove $_file"
+${KYLIN_HOME}/bin/kylin.sh 
org.apache.kylin.common.persistence.ResourceTool remove $_file
+
+elif [ "$1" == "cat" ]
+then
+
+_file=$2
+echo "Starting cat $_file"
+${KYLIN_HOME}/bin/kylin.sh 
org.apache.kylin.common.persistence.ResourceTool cat $_file
+
 elif [ "$1" == "reset" ]
 then
 
@@ -61,6 +82,9 @@ else
 echo "usage: metastore.sh backup"
 echo "   metastore.sh reset"
 echo "   metastore.sh restore PATH_TO_LOCAL_META"
+echo "   metastore.sh list RESOURCE_PATH"
+echo "   metastore.sh cat RESOURCE_PATH"
+echo "   metastore.sh remove RESOURCE_PATH"
 echo "   metastore.sh clean [--delete true]"
 exit 1
 fi

http://git-wip-us.apache.org/repos/asf/kylin/blob/17c33dc4/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
 
b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
index 489e45a..2e1d527 100644
--- 
a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
+++ 
b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
@@ -41,6 +41,9 @@ public class ResourceTool {
 System.out.println("Usage: ResourceTool list  RESOURCE_PATH");
 System.out.println("Usage: ResourceTool download  LOCAL_DIR");
 System.out.println("Usage: ResourceTool uploadLOCAL_DIR");
+System.out.println("Usage: ResourceTool reset");
+System.out.println("Usage: ResourceTool remove RESOURCE_PATH");
+System.out.println("Usage: ResourceTool cat RESOURCE_PATH");
 return;
 }

[kylin] Git Push Summary

2016-03-07 Thread mahongbin 
Repository: kylin
Updated Tags:  refs/tags/kylin-1.3 [created] d681d14cc

kylin git commit: [maven-release-plugin] prepare release kylin-1.3

2016-03-07 Thread mahongbin 
Repository: kylin
Updated Branches:
  refs/heads/v1.3-release f837d0e8c -> 99b50e1ce


[maven-release-plugin] prepare release kylin-1.3


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/99b50e1c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/99b50e1c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/99b50e1c

Branch: refs/heads/v1.3-release
Commit: 99b50e1ce517a1be8fe3e87108999d347aa4929b
Parents: f837d0e
Author: Hongbin Ma 
Authored: Mon Mar 7 08:52:07 2016 +
Committer: Hongbin Ma 
Committed: Mon Mar 7 08:52:07 2016 +

--
 atopcalcite/pom.xml   | 2 +-
 common/pom.xml| 2 +-
 cube/pom.xml  | 2 +-
 dictionary/pom.xml| 2 +-
 invertedindex/pom.xml | 2 +-
 jdbc/pom.xml  | 2 +-
 job/pom.xml   | 2 +-
 metadata/pom.xml  | 2 +-
 monitor/pom.xml   | 2 +-
 pom.xml   | 4 ++--
 query/pom.xml | 2 +-
 server/pom.xml| 2 +-
 storage/pom.xml   | 2 +-
 13 files changed, 14 insertions(+), 14 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/99b50e1c/atopcalcite/pom.xml
--
diff --git a/atopcalcite/pom.xml b/atopcalcite/pom.xml
index 03c926c..67f72ba 100644
--- a/atopcalcite/pom.xml
+++ b/atopcalcite/pom.xml
@@ -27,7 +27,7 @@
 
 org.apache.kylin
 kylin
-1.3-SNAPSHOT
+1.3
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/99b50e1c/common/pom.xml
--
diff --git a/common/pom.xml b/common/pom.xml
index 2f01bde..f636a95 100644
--- a/common/pom.xml
+++ b/common/pom.xml
@@ -27,7 +27,7 @@
 
 org.apache.kylin
 kylin
-1.3-SNAPSHOT
+1.3
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/99b50e1c/cube/pom.xml
--
diff --git a/cube/pom.xml b/cube/pom.xml
index b4fa9da..f2e46e9 100644
--- a/cube/pom.xml
+++ b/cube/pom.xml
@@ -27,7 +27,7 @@
 
 org.apache.kylin
 kylin
-1.3-SNAPSHOT
+1.3
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/99b50e1c/dictionary/pom.xml
--
diff --git a/dictionary/pom.xml b/dictionary/pom.xml
index ce86c66..2a6bef1 100644
--- a/dictionary/pom.xml
+++ b/dictionary/pom.xml
@@ -27,7 +27,7 @@
 
 org.apache.kylin
 kylin
-1.3-SNAPSHOT
+1.3
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/99b50e1c/invertedindex/pom.xml
--
diff --git a/invertedindex/pom.xml b/invertedindex/pom.xml
index 7fa3e52..2e9f0c2 100644
--- a/invertedindex/pom.xml
+++ b/invertedindex/pom.xml
@@ -21,7 +21,7 @@
 
 kylin
 org.apache.kylin
-1.3-SNAPSHOT
+1.3
 
 4.0.0
 kylin-invertedindex

http://git-wip-us.apache.org/repos/asf/kylin/blob/99b50e1c/jdbc/pom.xml
--
diff --git a/jdbc/pom.xml b/jdbc/pom.xml
index 6a1f836..db8917d 100644
--- a/jdbc/pom.xml
+++ b/jdbc/pom.xml
@@ -27,7 +27,7 @@

org.apache.kylin
kylin
-   1.3-SNAPSHOT
+   1.3

 


http://git-wip-us.apache.org/repos/asf/kylin/blob/99b50e1c/job/pom.xml
--
diff --git a/job/pom.xml b/job/pom.xml
index c62477e..8f427de 100644
--- a/job/pom.xml
+++ b/job/pom.xml
@@ -22,7 +22,7 @@
 
 org.apache.kylin
 kylin
-1.3-SNAPSHOT
+1.3
 
 
 kylin-job

http://git-wip-us.apache.org/repos/asf/kylin/blob/99b50e1c/metadata/pom.xml
--
diff --git a/metadata/pom.xml b/metadata/pom.xml
index 61b875c..515ed21 100644
--- a/metadata/pom.xml
+++ b/metadata/pom.xml
@@ -27,7 +27,7 @@
 
 org.apache.kylin
 kylin
-1.3-SNAPSHOT
+1.3
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/99b50e1c/monitor/pom.xml
--
diff --git a/monitor/pom.xml b/monitor/pom.xml
index 820934f..b1caab1 100644
--- a/monitor/pom.xml
+++ b/monitor/pom.xml
@@ -22,7 +22,7 @@
 
 org.apache.kylin
 kylin
-1.3-SNAPSHOT
+1.3
 
 
 4.0.0

http://git-wip-us.apache.org/repos/asf/kylin/blob/99b50e1c/pom.xml
--
diff --git a/pom.xml b/pom.xml
index 793a5f2..d76a9b3 100644
--- a/pom.xml

kylin git commit: [maven-release-plugin] prepare for next development iteration

2016-03-07 Thread mahongbin 
Repository: kylin
Updated Branches:
  refs/heads/v1.3-release 99b50e1ce -> 7d1cea66b


[maven-release-plugin] prepare for next development iteration


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7d1cea66
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7d1cea66
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7d1cea66

Branch: refs/heads/v1.3-release
Commit: 7d1cea66b08fd4386e21f195a30c772d98b1a59a
Parents: 99b50e1
Author: Hongbin Ma 
Authored: Mon Mar 7 08:53:06 2016 +
Committer: Hongbin Ma 
Committed: Mon Mar 7 08:53:06 2016 +

--
 atopcalcite/pom.xml   | 2 +-
 common/pom.xml| 2 +-
 cube/pom.xml  | 2 +-
 dictionary/pom.xml| 2 +-
 invertedindex/pom.xml | 2 +-
 jdbc/pom.xml  | 2 +-
 job/pom.xml   | 2 +-
 metadata/pom.xml  | 2 +-
 monitor/pom.xml   | 2 +-
 pom.xml   | 4 ++--
 query/pom.xml | 2 +-
 server/pom.xml| 2 +-
 storage/pom.xml   | 2 +-
 13 files changed, 14 insertions(+), 14 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/7d1cea66/atopcalcite/pom.xml
--
diff --git a/atopcalcite/pom.xml b/atopcalcite/pom.xml
index 67f72ba..fcb0283 100644
--- a/atopcalcite/pom.xml
+++ b/atopcalcite/pom.xml
@@ -27,7 +27,7 @@
 
 org.apache.kylin
 kylin
-1.3
+1.4-SNAPSHOT
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/7d1cea66/common/pom.xml
--
diff --git a/common/pom.xml b/common/pom.xml
index f636a95..aae60e4 100644
--- a/common/pom.xml
+++ b/common/pom.xml
@@ -27,7 +27,7 @@
 
 org.apache.kylin
 kylin
-1.3
+1.4-SNAPSHOT
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/7d1cea66/cube/pom.xml
--
diff --git a/cube/pom.xml b/cube/pom.xml
index f2e46e9..1bd8507 100644
--- a/cube/pom.xml
+++ b/cube/pom.xml
@@ -27,7 +27,7 @@
 
 org.apache.kylin
 kylin
-1.3
+1.4-SNAPSHOT
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/7d1cea66/dictionary/pom.xml
--
diff --git a/dictionary/pom.xml b/dictionary/pom.xml
index 2a6bef1..d26c0bf 100644
--- a/dictionary/pom.xml
+++ b/dictionary/pom.xml
@@ -27,7 +27,7 @@
 
 org.apache.kylin
 kylin
-1.3
+1.4-SNAPSHOT
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/7d1cea66/invertedindex/pom.xml
--
diff --git a/invertedindex/pom.xml b/invertedindex/pom.xml
index 2e9f0c2..488713c 100644
--- a/invertedindex/pom.xml
+++ b/invertedindex/pom.xml
@@ -21,7 +21,7 @@
 
 kylin
 org.apache.kylin
-1.3
+1.4-SNAPSHOT
 
 4.0.0
 kylin-invertedindex

http://git-wip-us.apache.org/repos/asf/kylin/blob/7d1cea66/jdbc/pom.xml
--
diff --git a/jdbc/pom.xml b/jdbc/pom.xml
index db8917d..1c19874 100644
--- a/jdbc/pom.xml
+++ b/jdbc/pom.xml
@@ -27,7 +27,7 @@

org.apache.kylin
kylin
-   1.3
+   1.4-SNAPSHOT

 


http://git-wip-us.apache.org/repos/asf/kylin/blob/7d1cea66/job/pom.xml
--
diff --git a/job/pom.xml b/job/pom.xml
index 8f427de..449f298 100644
--- a/job/pom.xml
+++ b/job/pom.xml
@@ -22,7 +22,7 @@
 
 org.apache.kylin
 kylin
-1.3
+1.4-SNAPSHOT
 
 
 kylin-job

http://git-wip-us.apache.org/repos/asf/kylin/blob/7d1cea66/metadata/pom.xml
--
diff --git a/metadata/pom.xml b/metadata/pom.xml
index 515ed21..68ec3d5 100644
--- a/metadata/pom.xml
+++ b/metadata/pom.xml
@@ -27,7 +27,7 @@
 
 org.apache.kylin
 kylin
-1.3
+1.4-SNAPSHOT
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/7d1cea66/monitor/pom.xml
--
diff --git a/monitor/pom.xml b/monitor/pom.xml
index b1caab1..16dffe4 100644
--- a/monitor/pom.xml
+++ b/monitor/pom.xml
@@ -22,7 +22,7 @@
 
 org.apache.kylin
 kylin
-1.3
+1.4-SNAPSHOT
 
 
 4.0.0

http://git-wip-us.apache.org/repos/asf/kylin/blob/7d1cea66/pom.xml
--
diff --git a/pom.xml b/pom.xml
index d76a9b3..e5d8416 100644

[kylin] Git Push Summary

2016-03-07 Thread mahongbin 
Repository: kylin
Updated Branches:
  refs/heads/v1.3-release [created] f837d0e8c