[42/50] [abbrv] kylin git commit: KYLIN-2773: fix join condition push down issue caused by type casting
KYLIN-2773: fix join condition push down issue caused by type casting Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/87f1732b Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/87f1732b Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/87f1732b Branch: refs/heads/master Commit: 87f1732b0a26191122788baaa2fc9d97c96210da Parents: 28ac7d9 Author: ZhongAuthored: Tue Aug 29 17:19:47 2017 +0800 Committer: Roger Shi Committed: Wed Aug 30 16:02:26 2017 +0800 -- .../calcite/sql2rel/SqlToRelConverter.java | 54 1 file changed, 54 insertions(+) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/87f1732b/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java -- diff --git a/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java b/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java index f10ba96..3615df7 100644 --- a/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java +++ b/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java @@ -2468,6 +2468,10 @@ public class SqlToRelConverter { return corr; } +if (containOnlyCast(joinCond)) { + joinCond = convertCastCondition(joinCond); +} + final Join originalJoin = (Join) RelFactories.DEFAULT_JOIN_FACTORY.createJoin(leftRel, rightRel, joinCond, ImmutableSet.of(), joinType, false); @@ -2475,6 +2479,56 @@ public class SqlToRelConverter { return RelOptUtil.pushDownJoinConditions(originalJoin); } + private boolean containOnlyCast(RexNode node) { +boolean result = true; +switch (node.getKind()) { + case AND: + case EQUALS: +final RexCall call = (RexCall) node; +List operands = Lists.newArrayList(call.getOperands()); +for (int i = 0; i < operands.size(); i++) { + RexNode operand = operands.get(i); + result &= containOnlyCast(operand); +} +break; + case OR: + case INPUT_REF: + case LITERAL: + case CAST: +return true; + default: +return false; +} +return result; + } + + private static RexNode convertCastCondition(RexNode node) { +switch (node.getKind()) { + case AND: + case EQUALS: +RexCall call = (RexCall) node; +List list = Lists.newArrayList(); +List operands = Lists.newArrayList(call.getOperands()); +for (int i = 0; i < operands.size(); i++) { + RexNode operand = operands.get(i); + final RexNode e = + convertCastCondition( + operand); + list.add(e); +} +if (!list.equals(call.getOperands())) { + return call.clone(call.getType(), list); +} +return call; + case CAST: +call = (RexCall) node; +operands = Lists.newArrayList(call.getOperands()); +return operands.get(0); + default: +return node; +} + } + private CorrelationUse getCorrelationUse(Blackboard bb, final RelNode r0) { final Set correlatedVariables = RelOptUtil.getVariablesUsed(r0);
[49/50] [abbrv] kylin git commit: Revert "#949 Refine column family for measures"
Revert "#949 Refine column family for measures" This reverts commit 906b8f40b381ab9086a4f7e7593ae7d8391189bd. Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/32d346b4 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/32d346b4 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/32d346b4 Branch: refs/heads/master Commit: 32d346b4cd619a38461a27115fbef2bd44f28372 Parents: b50030d Author: Hongbin MaAuthored: Thu Aug 31 22:41:20 2017 +0800 Committer: Hongbin Ma Committed: Thu Aug 31 22:41:20 2017 +0800 -- .../org/apache/kylin/gridtable/GTRecord.java | 19 +-- 1 file changed, 13 insertions(+), 6 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/32d346b4/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java -- diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java index 00e355f..d8c0fbd 100644 --- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java +++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java @@ -45,7 +45,7 @@ public class GTRecord implements Comparable, Cloneable { } this.info = info; } - + @Override public GTRecord clone() { // deep copy ByteArray[] cols = new ByteArray[this.cols.length]; @@ -114,6 +114,7 @@ public class GTRecord implements Comparable, Cloneable { return result; } + /** decode and return the values of this record */ public Object[] getValues(int[] selectedColumns, Object[] result) { assert selectedColumns.length <= result.length; @@ -294,21 +295,27 @@ public class GTRecord implements Comparable, Cloneable { buf.position(pos); } } - -public void loadColumnsFromColumnBlocks(ImmutableBitSet[] selectedColumnBlocks, ImmutableBitSet selectedCols, -ByteBuffer buf) { + +public void loadColumnsFromColumnBlocks(ImmutableBitSet[] selectedColumnBlocks, ImmutableBitSet selectedCols, ByteBuffer buf) { + int pos = buf.position(); + for (ImmutableBitSet selectedColBlock : selectedColumnBlocks) { + for (int i = 0; i < selectedColBlock.trueBitCount(); i++) { + int c = selectedColBlock.trueBitAt(i); + int len = info.codeSystem.codeLength(c, buf); -if (selectedCols.get(c)) { -cols[c].set(buf.array(), buf.arrayOffset() + pos, len); +if(selectedCols.get(c)) { +cols[c].set(buf.array(), buf.arrayOffset() + pos, len); } pos += len; buf.position(pos); } + } + } /** change pointers to point to data in given buffer, this
[46/50] [abbrv] kylin git commit: #949 Refine column family for measures
#949 Refine column family for measures Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a4aeb778 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a4aeb778 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a4aeb778 Branch: refs/heads/master Commit: a4aeb77890577e46300bc702c1117bf542081cc9 Parents: d62cb58 Author: auphyroc99 <454530...@qq.com> Authored: Wed Aug 30 16:03:24 2017 +0800 Committer: Roger ShiCommitted: Wed Aug 30 18:13:52 2017 +0800 -- .../org/apache/kylin/gridtable/GTRecord.java | 19 ++- 1 file changed, 6 insertions(+), 13 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/a4aeb778/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java -- diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java index 862fb99..c3a160a 100644 --- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java +++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java @@ -45,7 +45,7 @@ public class GTRecord implements Comparable, Cloneable { } this.info = info; } - + @Override public GTRecord clone() { // deep copy ByteArray[] cols = new ByteArray[this.cols.length]; @@ -114,7 +114,6 @@ public class GTRecord implements Comparable, Cloneable { return result; } - /** decode and return the values of this record */ public Object[] getValues(int[] selectedColumns, Object[] result) { assert selectedColumns.length <= result.length; @@ -295,27 +294,21 @@ public class GTRecord implements Comparable, Cloneable { buf.position(pos); } } - -public void loadColumnsFromColumnBlocks(ImmutableBitSet[] selectedColumnBlocks, ImmutableBitSet selectedCols, ByteBuffer buf) { - + +public void loadColumnsFromColumnBlocks(ImmutableBitSet[] selectedColumnBlocks, ImmutableBitSet selectedCols, +ByteBuffer buf) { int pos = buf.position(); - for (ImmutableBitSet selectedColBlock : selectedColumnBlocks) { - for (int i = 0; i < selectedColBlock.trueBitCount(); i++) { - int c = selectedColBlock.trueBitAt(i); - int len = info.codeSystem.codeLength(c, buf); -if(selectedCols.get(c)) { -cols[c].set(buf.array(), buf.arrayOffset() + pos, len); +if (selectedCols.get(c)) { +cols[c].set(buf.array(), buf.arrayOffset() + pos, len); } pos += len; buf.position(pos); } - } - } }
[38/50] [abbrv] kylin git commit: KYLIN-2803 Pushdown non select query
KYLIN-2803 Pushdown non select query Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/dbecb9b8 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/dbecb9b8 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/dbecb9b8 Branch: refs/heads/master Commit: dbecb9b84bc90049acb3ef314128ac71f3516731 Parents: 480592d Author: shi shaofengAuthored: Fri Aug 25 16:18:21 2017 +0800 Committer: SHAOFENG SHI Committed: Mon Aug 28 16:02:43 2017 +0800 -- .../apache/kylin/common/KylinConfigBase.java| 4 ++ .../adhocquery/HivePushDownConverter.java | 4 ++ .../source/adhocquery/IPushDownRunner.java | 13 ++ .../query/adhoc/PushDownRunnerJdbcImpl.java | 36 - .../apache/kylin/query/util/PushDownUtil.java | 32 ++- .../org/apache/kylin/query/util/QueryUtil.java | 28 + .../apache/kylin/query/util/QueryUtilTest.java | 39 ++ .../apache/kylin/rest/service/QueryService.java | 42 +--- 8 files changed, 172 insertions(+), 26 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/dbecb9b8/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java -- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index bca00e7..a113327 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -1088,6 +1088,10 @@ abstract public class KylinConfigBase implements Serializable { new String[] { "org.apache.kylin.source.adhocquery.HivePushDownConverter" }); } +public boolean isPushdownQueryCacheEnabled() { +return Boolean.parseBoolean(this.getOptional("kylin.query.pushdown.cache-enabled", "false")); +} + public String getJdbcUrl() { return getOptional("kylin.query.pushdown.jdbc.url", ""); } http://git-wip-us.apache.org/repos/asf/kylin/blob/dbecb9b8/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/HivePushDownConverter.java -- diff --git a/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/HivePushDownConverter.java b/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/HivePushDownConverter.java index bcd8608..eef4594 100644 --- a/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/HivePushDownConverter.java +++ b/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/HivePushDownConverter.java @@ -262,6 +262,10 @@ public class HivePushDownConverter implements IPushDownConverter { // Step7.Add quote for interval in timestampadd convertedSql = timestampaddReplace(convertedSql); +// Step8.Replace integer with int +convertedSql = replaceString(convertedSql, "INTEGER", "INT"); +convertedSql = replaceString(convertedSql, "integer", "int"); + return convertedSql; } http://git-wip-us.apache.org/repos/asf/kylin/blob/dbecb9b8/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/IPushDownRunner.java -- diff --git a/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/IPushDownRunner.java b/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/IPushDownRunner.java index c8d18aa..0336bfb 100644 --- a/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/IPushDownRunner.java +++ b/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/IPushDownRunner.java @@ -36,4 +36,17 @@ public interface IPushDownRunner { * @throws Exception if running pushdown query fails */ void executeQuery(String query, List returnRows, List returnColumnMeta) throws Exception; + + +/** + * Run an pushdown non-query sql + * + * @param sql the sql statement + * + * @return whether the SQL is executed successfully + * + * @throws Exception if running pushdown fails + */ +boolean executeUpdate(String sql) throws Exception; + } http://git-wip-us.apache.org/repos/asf/kylin/blob/dbecb9b8/query/src/main/java/org/apache/kylin/query/adhoc/PushDownRunnerJdbcImpl.java -- diff --git a/query/src/main/java/org/apache/kylin/query/adhoc/PushDownRunnerJdbcImpl.java b/query/src/main/java/org/apache/kylin/query/adhoc/PushDownRunnerJdbcImpl.java index 751e6b0..713629a 100644 ---
[50/50] [abbrv] kylin git commit: Revert "KYLIN-2790 Add extending point for column family"
Revert "KYLIN-2790 Add extending point for column family" This reverts commit 842dc4ca1dabb576aed40c55ebfc1cbabaaf3944. Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/363f858b Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/363f858b Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/363f858b Branch: refs/heads/master Commit: 363f858b890ea04c87ee7b081fc74805766e0b93 Parents: 32d346b Author: Hongbin MaAuthored: Thu Aug 31 22:43:36 2017 +0800 Committer: Hongbin Ma Committed: Thu Aug 31 22:43:36 2017 +0800 -- .../apache/kylin/common/KylinConfigBase.java| 4 .../org/apache/kylin/cube/model/CubeDesc.java | 18 ++-- .../kylin/cube/model/HBaseMappingDesc.java | 18 .../org/apache/kylin/gridtable/GTRecord.java| 22 4 files changed, 2 insertions(+), 60 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/363f858b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java -- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index d32f08c..0e990df 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -286,10 +286,6 @@ abstract public class KylinConfigBase implements Serializable { "org.apache.kylin.storage.hbase.util.ZookeeperDistributedLock$Factory"); return (DistributedLockFactory) ClassUtil.newInstance(clsName); } - -public String getHBaseMappingAdapter() { -return getOptional("kylin.metadata.hbasemapping-adapter"); -} // // DICTIONARY & SNAPSHOT http://git-wip-us.apache.org/repos/asf/kylin/blob/363f858b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java -- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java index d263e1f..0e22587 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java @@ -22,7 +22,6 @@ import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; import static com.google.common.base.Preconditions.checkState; -import java.lang.reflect.Method; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.util.ArrayList; @@ -604,21 +603,8 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware { validateAggregationGroups(); // check if aggregation group is valid validateAggregationGroupsCombination(); -String hbaseMappingAdapterName = config.getHBaseMappingAdapter(); - -if (hbaseMappingAdapterName != null) { -try { -Class hbaseMappingAdapterClass = Class.forName(hbaseMappingAdapterName); -Method initMethod = hbaseMappingAdapterClass.getMethod("initHBaseMapping", CubeDesc.class); -initMethod.invoke(null, this); -} catch (Exception e) { -logger.error("Wrong configuration for kylin.metadata.hbasemapping-adapter: class " -+ hbaseMappingAdapterName + " not found. "); -} -} else { -if (hbaseMapping != null) { -hbaseMapping.init(this); -} +if (hbaseMapping != null) { +hbaseMapping.init(this); } initMeasureReferenceToColumnFamily(); http://git-wip-us.apache.org/repos/asf/kylin/blob/363f858b/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseMappingDesc.java -- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseMappingDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseMappingDesc.java index 77b4b14..d1e5829 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseMappingDesc.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseMappingDesc.java @@ -87,24 +87,6 @@ public class HBaseMappingDesc implements java.io.Serializable { } } -public void initAsSeparatedColumns(CubeDesc cubeDesc) { -cubeRef = cubeDesc; - -int cfNum = cubeDesc.getMeasures().size(); -columnFamily = new
[40/50] [abbrv] kylin git commit: Revert "KYLIN-2816 wrap columns with backtick"
Revert "KYLIN-2816 wrap columns with backtick" This reverts commit 53b25c6137731c1b0d26925be7dcaa0252d75b64. Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/81e9f573 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/81e9f573 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/81e9f573 Branch: refs/heads/master Commit: 81e9f573fae2af9511ac6004f69101598fc7a71a Parents: dbecb9b Author: Hongbin MaAuthored: Mon Aug 28 19:19:10 2017 +0800 Committer: Hongbin Ma Committed: Tue Aug 29 10:36:10 2017 +0800 -- .../org/apache/kylin/job/JoinedFlatTable.java | 77 ++-- .../kylin/job/TestWrapColumnWithBacktick.java | 57 --- .../metadata/model/tool/CalciteParser.java | 8 +- 3 files changed, 7 insertions(+), 135 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/81e9f573/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java -- diff --git a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java index 4ac5a3a..824c693 100644 --- a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java +++ b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java @@ -21,17 +21,11 @@ package org.apache.kylin.job; import java.lang.reflect.Method; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; -import java.util.Comparator; import java.util.HashSet; import java.util.List; import java.util.Set; -import org.apache.calcite.sql.SqlIdentifier; -import org.apache.calcite.sql.SqlNode; -import org.apache.calcite.sql.util.SqlBasicVisitor; import org.apache.kylin.common.KylinConfig; -import org.apache.kylin.common.util.Pair; import org.apache.kylin.cube.CubeSegment; import org.apache.kylin.job.engine.JobEngineConfig; import org.apache.kylin.metadata.model.DataModelDesc; @@ -41,9 +35,6 @@ import org.apache.kylin.metadata.model.JoinTableDesc; import org.apache.kylin.metadata.model.PartitionDesc; import org.apache.kylin.metadata.model.TableRef; import org.apache.kylin.metadata.model.TblColRef; -import org.apache.kylin.metadata.model.tool.CalciteParser; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** * @@ -52,7 +43,6 @@ import org.slf4j.LoggerFactory; public class JoinedFlatTable { public static final String BACKTICK = "`"; -private static final Logger logger = LoggerFactory.getLogger(JoinedFlatTable.class); public static String getTableDir(IJoinedFlatTableDesc flatDesc, String storageDfsDir) { return storageDfsDir + "/" + flatDesc.getTableName(); @@ -140,7 +130,11 @@ public class JoinedFlatTable { sql.append(","); } String colTotalName = String.format("%s.%s", col.getTableRef().getTableName(), col.getName()); -String expressionInSourceDB = quoteOriginalColumnWithBacktick(col.getExpressionInSourceDB()); +String expressionInSourceDB = col.getExpressionInSourceDB(); +if (expressionInSourceDB.contains(".")) { +// surround column name with back-tick, to support unicode column name +expressionInSourceDB = expressionInSourceDB.replace(".", "." + BACKTICK) + BACKTICK; +} if (skipAsList.contains(colTotalName)) { sql.append(expressionInSourceDB + sep); } else { @@ -283,65 +277,4 @@ public class JoinedFlatTable { return sql.toString(); } -/** - * quote column name with back-tick, to support unicode column name - * - * @param sourceSQL - * @return - */ -static String quoteOriginalColumnWithBacktick(String sourceSQL) { -StringBuilder result = new StringBuilder(sourceSQL); -try { -List > replacePoses = new ArrayList<>(); -for (SqlIdentifier id : SqlIdentifierVisitor.getAllSqlIdentifier(CalciteParser.getExpNode(sourceSQL))) { - replacePoses.add(CalciteParser.getReplacePos(id.getComponentParserPosition(1), "select " + sourceSQL + " from t")); -} - -// latter replace position in the front of the list. -Collections.sort(replacePoses, new Comparator >() { -@Override -public int compare(Pair o1, Pair o2) { -return -(o1.getSecond() - o2.getSecond()); -} -}); - -// cuz the pos is add "select "'s size, so minus 7 offset. -final int OFFSET = 7; -for (Pair replacePos :
[44/50] [abbrv] kylin git commit: KYLIN-2823 Trim TupleFilter after dictionary-based filter optimization
KYLIN-2823 Trim TupleFilter after dictionary-based filter optimization Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/47822891 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/47822891 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/47822891 Branch: refs/heads/master Commit: 47822891332aef146ae3c2b5e3adc98c23024770 Parents: 1eb3617 Author: Hongbin MaAuthored: Wed Aug 30 16:06:18 2017 +0800 Committer: æ Committed: Wed Aug 30 17:40:03 2017 +0800 -- .../java/org/apache/kylin/gridtable/GTUtil.java | 24 +++- .../metadata/filter/ConstantTupleFilter.java| 14 +++ .../filter/FilterOptimizeTransformer.java | 35 +- .../filter/FilterOptimizeTransformerTest.java | 121 +++ .../storage/gtrecord/DictGridTableTest.java | 108 +++-- .../src/test/resources/query/sql/query106.sql | 101 .../src/test/resources/query/sql/query107.sql | 102 7 files changed, 459 insertions(+), 46 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/47822891/core-cube/src/main/java/org/apache/kylin/gridtable/GTUtil.java -- diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTUtil.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTUtil.java index 7a7e4e6..5c9dfe3 100755 --- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTUtil.java +++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTUtil.java @@ -30,6 +30,7 @@ import org.apache.kylin.common.util.BytesUtil; import org.apache.kylin.metadata.filter.ColumnTupleFilter; import org.apache.kylin.metadata.filter.CompareTupleFilter; import org.apache.kylin.metadata.filter.ConstantTupleFilter; +import org.apache.kylin.metadata.filter.FilterOptimizeTransformer; import org.apache.kylin.metadata.filter.IFilterCodeSystem; import org.apache.kylin.metadata.filter.TupleFilter; import org.apache.kylin.metadata.filter.TupleFilterSerializer; @@ -64,7 +65,15 @@ public class GTUtil { public static TupleFilter convertFilterColumnsAndConstants(TupleFilter rootFilter, GTInfo info, // List colMapping, Set unevaluatableColumnCollector) { Map map = colListToMap(colMapping); -return convertFilter(rootFilter, info, map, true, unevaluatableColumnCollector); +TupleFilter filter = convertFilter(rootFilter, info, map, true, unevaluatableColumnCollector); + +// optimize the filter: after translating with dictionary, some filters become determined +// e.g. +// ( a = 'value_in_dict' OR a = 'value_not_in_dict') will become (a = 'value_in_dict' OR ConstantTupleFilter.FALSE) +// use the following to further trim the filter to (a = 'value_in_dict') +// The goal is to avoid too many children after flatten filter step +filter = new FilterOptimizeTransformer().transform(filter); +return filter; } protected static Map colListToMap(List colMapping) { @@ -81,8 +90,9 @@ public class GTUtil { final Set unevaluatableColumnCollector) { IFilterCodeSystem filterCodeSystem = wrap(info.codeSystem.getComparator()); - -GTConvertDecorator decorator = new GTConvertDecorator(unevaluatableColumnCollector, colMapping, info, encodeConstants); + +GTConvertDecorator decorator = new GTConvertDecorator(unevaluatableColumnCollector, colMapping, info, +encodeConstants); byte[] bytes = TupleFilterSerializer.serialize(rootFilter, decorator, filterCodeSystem); return TupleFilterSerializer.deserialize(bytes, filterCodeSystem); @@ -122,14 +132,15 @@ public class GTUtil { protected final GTInfo info; protected final boolean encodeConstants; -public GTConvertDecorator(Set unevaluatableColumnCollector, Map colMapping, GTInfo info, boolean encodeConstants) { +public GTConvertDecorator(Set unevaluatableColumnCollector, Map colMapping, +GTInfo info, boolean encodeConstants) { this.unevaluatableColumnCollector = unevaluatableColumnCollector; this.colMapping = colMapping; this.info = info; this.encodeConstants = encodeConstants; buf = ByteBuffer.allocate(info.getMaxColumnLength()); } - + protected int mapCol(TblColRef col) { Integer i = colMapping.get(col); return i == null ? -1 : i; @@ -143,7 +154,8 @@ public class GTUtil { // In case of NOT(unEvaluatableFilter), we should immediately replace it as TRUE, //
[33/50] [abbrv] kylin git commit: minor, special treatment for OLAPContext.allColumns when select star
minor, special treatment for OLAPContext.allColumns when select star Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3200d6a1 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3200d6a1 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3200d6a1 Branch: refs/heads/master Commit: 3200d6a148d4368801e415f6ae969bbab7c7345e Parents: 28a07eb Author: Hongbin MaAuthored: Sun Aug 27 15:41:22 2017 +0800 Committer: Hongbin Ma Committed: Mon Aug 28 10:42:57 2017 +0800 -- .../org/apache/kylin/query/relnode/OLAPTableScan.java | 13 +++-- 1 file changed, 11 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/3200d6a1/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java -- diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java index 7801891..dd2160b 100644 --- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java +++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java @@ -210,7 +210,8 @@ public class OLAPTableScan extends TableScan implements OLAPRel, EnumerableRel { Preconditions.checkState(columnRowType == null, "OLAPTableScan MUST NOT be shared by more than one prent"); // create context in case of non-join -if (implementor.getContext() == null || !(implementor.getParentNode() instanceof OLAPJoinRel) || implementor.isNewOLAPContextRequired()) { +if (implementor.getContext() == null || !(implementor.getParentNode() instanceof OLAPJoinRel) +|| implementor.isNewOLAPContextRequired()) { implementor.allocateContext(); } @@ -227,6 +228,13 @@ public class OLAPTableScan extends TableScan implements OLAPRel, EnumerableRel { if (context.firstTableScan == null) { context.firstTableScan = this; } + +if (implementor.getParentNode() instanceof OLAPToEnumerableConverter) { +// OLAPToEnumerableConverter on top of table scan, should be a select * from table +for (TblColRef tblColRef : columnRowType.getAllColumns()) { +context.allColumns.add(tblColRef); +} +} } public String getAlias() { @@ -288,7 +296,8 @@ public class OLAPTableScan extends TableScan implements OLAPRel, EnumerableRel { String execFunction = genExecFunc(); PhysType physType = PhysTypeImpl.of(implementor.getTypeFactory(), this.rowType, pref.preferArray()); -MethodCallExpression exprCall = Expressions.call(table.getExpression(OLAPTable.class), execFunction, implementor.getRootExpression(), Expressions.constant(context.id)); +MethodCallExpression exprCall = Expressions.call(table.getExpression(OLAPTable.class), execFunction, +implementor.getRootExpression(), Expressions.constant(context.id)); return implementor.result(physType, Blocks.toBlock(exprCall)); }
[36/50] [abbrv] kylin git commit: KYLIN-2816 wrap columns with backtick
KYLIN-2816 wrap columns with backtick Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/74c904e4 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/74c904e4 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/74c904e4 Branch: refs/heads/master Commit: 74c904e4eedc224b07c9b6463ab0b63faddd4a21 Parents: b778f33 Author: tttMelody <245915...@qq.com> Authored: Mon Aug 28 14:19:36 2017 +0800 Committer: Hongbin MaCommitted: Mon Aug 28 15:25:27 2017 +0800 -- .../org/apache/kylin/job/JoinedFlatTable.java | 77 ++-- .../kylin/job/TestWrapColumnWithBacktick.java | 57 +++ .../metadata/model/tool/CalciteParser.java | 8 +- 3 files changed, 135 insertions(+), 7 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/74c904e4/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java -- diff --git a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java index 824c693..4ac5a3a 100644 --- a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java +++ b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java @@ -21,11 +21,17 @@ package org.apache.kylin.job; import java.lang.reflect.Method; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; import java.util.HashSet; import java.util.List; import java.util.Set; +import org.apache.calcite.sql.SqlIdentifier; +import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.util.SqlBasicVisitor; import org.apache.kylin.common.KylinConfig; +import org.apache.kylin.common.util.Pair; import org.apache.kylin.cube.CubeSegment; import org.apache.kylin.job.engine.JobEngineConfig; import org.apache.kylin.metadata.model.DataModelDesc; @@ -35,6 +41,9 @@ import org.apache.kylin.metadata.model.JoinTableDesc; import org.apache.kylin.metadata.model.PartitionDesc; import org.apache.kylin.metadata.model.TableRef; import org.apache.kylin.metadata.model.TblColRef; +import org.apache.kylin.metadata.model.tool.CalciteParser; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * @@ -43,6 +52,7 @@ import org.apache.kylin.metadata.model.TblColRef; public class JoinedFlatTable { public static final String BACKTICK = "`"; +private static final Logger logger = LoggerFactory.getLogger(JoinedFlatTable.class); public static String getTableDir(IJoinedFlatTableDesc flatDesc, String storageDfsDir) { return storageDfsDir + "/" + flatDesc.getTableName(); @@ -130,11 +140,7 @@ public class JoinedFlatTable { sql.append(","); } String colTotalName = String.format("%s.%s", col.getTableRef().getTableName(), col.getName()); -String expressionInSourceDB = col.getExpressionInSourceDB(); -if (expressionInSourceDB.contains(".")) { -// surround column name with back-tick, to support unicode column name -expressionInSourceDB = expressionInSourceDB.replace(".", "." + BACKTICK) + BACKTICK; -} +String expressionInSourceDB = quoteOriginalColumnWithBacktick(col.getExpressionInSourceDB()); if (skipAsList.contains(colTotalName)) { sql.append(expressionInSourceDB + sep); } else { @@ -277,4 +283,65 @@ public class JoinedFlatTable { return sql.toString(); } +/** + * quote column name with back-tick, to support unicode column name + * + * @param sourceSQL + * @return + */ +static String quoteOriginalColumnWithBacktick(String sourceSQL) { +StringBuilder result = new StringBuilder(sourceSQL); +try { +List > replacePoses = new ArrayList<>(); +for (SqlIdentifier id : SqlIdentifierVisitor.getAllSqlIdentifier(CalciteParser.getExpNode(sourceSQL))) { + replacePoses.add(CalciteParser.getReplacePos(id.getComponentParserPosition(1), "select " + sourceSQL + " from t")); +} + +// latter replace position in the front of the list. +Collections.sort(replacePoses, new Comparator >() { +@Override +public int compare(Pair o1, Pair o2) { +return -(o1.getSecond() - o2.getSecond()); +} +}); + +// cuz the pos is add "select "'s size, so minus 7 offset. +final int OFFSET = 7; +for (Pair replacePos : replacePoses) { +result.insert(replacePos.getSecond()
[01/50] [abbrv] kylin git commit: KYLIN-2785 Handle Maprfs schema working directory
Repository: kylin Updated Branches: refs/heads/master 727920b4d -> 363f858b8 KYLIN-2785 Handle Maprfs schema working directory Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5008866f Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5008866f Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5008866f Branch: refs/heads/master Commit: 5008866f41efd13ce5abaf8cc19c09919fe698dd Parents: 0153263 Author: Roger ShiAuthored: Thu Aug 10 21:34:03 2017 +0800 Committer: Roger Shi Committed: Sun Aug 13 22:39:54 2017 +0800 -- .../src/main/java/org/apache/kylin/common/KylinConfigBase.java | 2 ++ 1 file changed, 2 insertions(+) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/5008866f/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java -- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index 3a06571..78abf22 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -220,6 +220,8 @@ abstract public class KylinConfigBase implements Serializable { cachedHdfsWorkingDirectory = root; if (cachedHdfsWorkingDirectory.startsWith("file:")) { cachedHdfsWorkingDirectory = cachedHdfsWorkingDirectory.replace("file:", "file://"); +} else if (cachedHdfsWorkingDirectory.startsWith("maprfs:")) { +cachedHdfsWorkingDirectory = cachedHdfsWorkingDirectory.replace("maprfs:", "maprfs://"); } return cachedHdfsWorkingDirectory; }
[18/50] [abbrv] kylin git commit: minor fix CI
minor fix CI Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/46cd8813 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/46cd8813 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/46cd8813 Branch: refs/heads/master Commit: 46cd8813902881912aaed9969151092b48f49d57 Parents: 909628f Author: Hongbin MaAuthored: Tue Aug 22 11:16:41 2017 +0800 Committer: Hongbin Ma Committed: Tue Aug 22 11:41:31 2017 +0800 -- .../query/sql_computedcolumn/query04.sql| 28 +++- .../sql_computedcolumn/query04.sql.compare | 28 ++-- 2 files changed, 53 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/46cd8813/kylin-it/src/test/resources/query/sql_computedcolumn/query04.sql -- diff --git a/kylin-it/src/test/resources/query/sql_computedcolumn/query04.sql b/kylin-it/src/test/resources/query/sql_computedcolumn/query04.sql index 780de5d..75af19b 100644 --- a/kylin-it/src/test/resources/query/sql_computedcolumn/query04.sql +++ b/kylin-it/src/test/resources/query/sql_computedcolumn/query04.sql @@ -1 +1,27 @@ -select count(*) as a , sum (F.DEAL_AMOUNT) as b from test_kylin_fact f inner join test_account a on f.seller_id = a.account_id inner join test_country c on country = account_country group by F.SELLER_ID_AND_COUNTRY_NAME +select count(*) as a , sum (TEST_KYLIN_FACT.DEAL_AMOUNT) as b , TEST_KYLIN_FACT.SELLER_ID_AND_COUNTRY_NAME as c + + + + +FROM TEST_KYLIN_FACT as TEST_KYLIN_FACT +INNER JOIN TEST_ORDER as TEST_ORDER +ON TEST_KYLIN_FACT.ORDER_ID = TEST_ORDER.ORDER_ID +INNER JOIN TEST_ACCOUNT as BUYER_ACCOUNT +ON TEST_ORDER.BUYER_ID = BUYER_ACCOUNT.ACCOUNT_ID +INNER JOIN TEST_ACCOUNT as SELLER_ACCOUNT +ON TEST_KYLIN_FACT.SELLER_ID = SELLER_ACCOUNT.ACCOUNT_ID +INNER JOIN TEST_CATEGORY_GROUPINGS as TEST_CATEGORY_GROUPINGS +ON TEST_KYLIN_FACT.LEAF_CATEG_ID = TEST_CATEGORY_GROUPINGS.LEAF_CATEG_ID AND TEST_KYLIN_FACT.LSTG_SITE_ID = TEST_CATEGORY_GROUPINGS.SITE_ID +INNER JOIN TEST_COUNTRY as BUYER_COUNTRY +ON BUYER_ACCOUNT.ACCOUNT_COUNTRY = BUYER_COUNTRY.COUNTRY +INNER JOIN TEST_COUNTRY as SELLER_COUNTRY +ON SELLER_ACCOUNT.ACCOUNT_COUNTRY = SELLER_COUNTRY.COUNTRY + + +group by TEST_KYLIN_FACT.SELLER_ID_AND_COUNTRY_NAME + +order by b + + + + http://git-wip-us.apache.org/repos/asf/kylin/blob/46cd8813/kylin-it/src/test/resources/query/sql_computedcolumn/query04.sql.compare -- diff --git a/kylin-it/src/test/resources/query/sql_computedcolumn/query04.sql.compare b/kylin-it/src/test/resources/query/sql_computedcolumn/query04.sql.compare index 5ee361d..66ffc07 100644 --- a/kylin-it/src/test/resources/query/sql_computedcolumn/query04.sql.compare +++ b/kylin-it/src/test/resources/query/sql_computedcolumn/query04.sql.compare @@ -1,2 +1,26 @@ -select count(*) as a, sum (F.PRICE * F.ITEM_COUNT) as b from test_kylin_fact f -inner join test_account a on f.seller_id = a.account_id inner join test_country c on country = account_country group by CONCAT(A.ACCOUNT_ID, C.NAME) +select count(*) as a, sum (TEST_KYLIN_FACT.PRICE * TEST_KYLIN_FACT.ITEM_COUNT) as b, CONCAT(SELLER_ACCOUNT.ACCOUNT_ID, SELLER_COUNTRY.NAME) as c + + + +FROM TEST_KYLIN_FACT as TEST_KYLIN_FACT +INNER JOIN TEST_ORDER as TEST_ORDER +ON TEST_KYLIN_FACT.ORDER_ID = TEST_ORDER.ORDER_ID +INNER JOIN TEST_ACCOUNT as BUYER_ACCOUNT +ON TEST_ORDER.BUYER_ID = BUYER_ACCOUNT.ACCOUNT_ID +INNER JOIN TEST_ACCOUNT as SELLER_ACCOUNT +ON TEST_KYLIN_FACT.SELLER_ID = SELLER_ACCOUNT.ACCOUNT_ID +INNER JOIN TEST_CATEGORY_GROUPINGS as TEST_CATEGORY_GROUPINGS +ON TEST_KYLIN_FACT.LEAF_CATEG_ID = TEST_CATEGORY_GROUPINGS.LEAF_CATEG_ID AND TEST_KYLIN_FACT.LSTG_SITE_ID = TEST_CATEGORY_GROUPINGS.SITE_ID +INNER JOIN TEST_COUNTRY as BUYER_COUNTRY +ON BUYER_ACCOUNT.ACCOUNT_COUNTRY = BUYER_COUNTRY.COUNTRY +INNER JOIN TEST_COUNTRY as SELLER_COUNTRY +ON SELLER_ACCOUNT.ACCOUNT_COUNTRY = SELLER_COUNTRY.COUNTRY + + + + + + + +group by CONCAT(SELLER_ACCOUNT.ACCOUNT_ID, SELLER_COUNTRY.NAME) +order by b
[43/50] [abbrv] kylin git commit: KYLIN-2773 Test for join casting
KYLIN-2773 Test for join casting Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1eb3617c Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1eb3617c Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1eb3617c Branch: refs/heads/master Commit: 1eb3617cf1cf50da16d5f86b4f922b3e8925de4b Parents: 87f1732 Author: Roger ShiAuthored: Tue Aug 29 19:22:18 2017 +0800 Committer: Roger Shi Committed: Wed Aug 30 16:02:26 2017 +0800 -- .../table/DEFAULT.TEST_KYLIN_FACT.json | 2 +- .../apache/kylin/query/ITKylinQueryTest.java| 5 + .../test/resources/query/sql_join/query_00.sql | 21 3 files changed, 27 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/1eb3617c/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json -- diff --git a/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json b/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json index e82d535..4be05f2 100644 --- a/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json +++ b/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json @@ -44,7 +44,7 @@ }, { "id" : "8", "name" : "SELLER_ID", -"datatype" : "bigint", +"datatype" : "int", "data_gen" : "RAND||1000|10001000", "index": "T" }, { http://git-wip-us.apache.org/repos/asf/kylin/blob/1eb3617c/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java -- diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java index 4ea4497..e332cda 100644 --- a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java +++ b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java @@ -234,6 +234,11 @@ public class ITKylinQueryTest extends KylinTestBase { } @Test +public void testJoinCastQuery() throws Exception { +execAndCompQuery(getQueryFolderPrefix() + "src/test/resources/query/sql_join", null, true); +} + +@Test public void testCachedQuery() throws Exception { execAndCompQuery(getQueryFolderPrefix() + "src/test/resources/query/sql_cache", null, true); } http://git-wip-us.apache.org/repos/asf/kylin/blob/1eb3617c/kylin-it/src/test/resources/query/sql_join/query_00.sql -- diff --git a/kylin-it/src/test/resources/query/sql_join/query_00.sql b/kylin-it/src/test/resources/query/sql_join/query_00.sql new file mode 100644 index 000..024e45d --- /dev/null +++ b/kylin-it/src/test/resources/query/sql_join/query_00.sql @@ -0,0 +1,21 @@ +-- join condition cast type +select sum(ITEM_COUNT) as ITEM_CNT +FROM TEST_KYLIN_FACT as TEST_KYLIN_FACT +INNER JOIN TEST_ORDER as TEST_ORDER +ON TEST_KYLIN_FACT.ORDER_ID = TEST_ORDER.ORDER_ID +INNER JOIN TEST_ACCOUNT as BUYER_ACCOUNT +ON TEST_ORDER.BUYER_ID = BUYER_ACCOUNT.ACCOUNT_ID +INNER JOIN TEST_ACCOUNT as SELLER_ACCOUNT +ON TEST_KYLIN_FACT.SELLER_ID = SELLER_ACCOUNT.ACCOUNT_ID +INNER JOIN EDW.TEST_CAL_DT as TEST_CAL_DT +ON TEST_KYLIN_FACT.CAL_DT = TEST_CAL_DT.CAL_DT +INNER JOIN TEST_CATEGORY_GROUPINGS as TEST_CATEGORY_GROUPINGS +ON TEST_KYLIN_FACT.LEAF_CATEG_ID = TEST_CATEGORY_GROUPINGS.LEAF_CATEG_ID AND TEST_KYLIN_FACT.LSTG_SITE_ID = TEST_CATEGORY_GROUPINGS.SITE_ID +INNER JOIN EDW.TEST_SITES as TEST_SITES +ON TEST_KYLIN_FACT.LSTG_SITE_ID = TEST_SITES.SITE_ID +INNER JOIN EDW.TEST_SELLER_TYPE_DIM as TEST_SELLER_TYPE_DIM +ON TEST_KYLIN_FACT.SLR_SEGMENT_CD = TEST_SELLER_TYPE_DIM.SELLER_TYPE_CD +INNER JOIN TEST_COUNTRY as BUYER_COUNTRY +ON BUYER_ACCOUNT.ACCOUNT_COUNTRY = BUYER_COUNTRY.COUNTRY +INNER JOIN TEST_COUNTRY as SELLER_COUNTRY +ON SELLER_ACCOUNT.ACCOUNT_COUNTRY = SELLER_COUNTRY.COUNTRY \ No newline at end of file
[35/50] [abbrv] kylin git commit: KYLIN-2776 Using dropwizard as default metric framework modify comment code
KYLIN-2776 Using dropwizard as default metric framework modify comment code Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b778f33a Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b778f33a Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b778f33a Branch: refs/heads/master Commit: b778f33af3718fd1bf5f705b81104f424803fe78 Parents: fad75c9 Author: yiming.xu <100650...@qq.com> Authored: Mon Aug 28 12:57:10 2017 +0800 Committer: Hongbin MaCommitted: Mon Aug 28 13:50:36 2017 +0800 -- .../apache/kylin/common/KylinConfigBase.java| 4 ++ .../kylin/common/metrics/common/Metrics.java| 40 --- .../common/metrics/common/MetricsConstant.java | 2 +- .../common/metrics/common/MetricsFactory.java | 2 +- .../metrics/common/MetricsNameBuilder.java | 38 ++ .../common/metrics/common/MetricsScope.java | 8 +-- .../kylin/common/metrics/common/Metricss.java | 40 --- .../metrics/metrics2/CodahaleMetrics.java | 20 .../metrics2/ConsoleMetricsReporter.java| 2 +- .../metrics/metrics2/JmxMetricsReporter.java| 2 +- .../metrics2/MetricVariableRatioGauge.java | 2 +- .../metrics/metrics2/Metrics2Reporter.java | 2 +- .../metrics/metrics2/MetricsReporting.java | 3 -- .../common/metrics/perflog/PerfLogger.java | 3 +- .../kylin/rest/metrics/QueryMetrics2Facade.java | 20 .../kylin/rest/metrics/QueryMetrics2Test.java | 52 ++-- 16 files changed, 122 insertions(+), 118 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/b778f33a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java -- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index 99b9bb2..bca00e7 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -1149,6 +1149,10 @@ abstract public class KylinConfigBase implements Serializable { return Boolean.parseBoolean(getOptional("kylin.server.query-metrics-enabled", "false")); } +public boolean getQueryMetrics2Enabled() { +return Boolean.parseBoolean(getOptional("kylin.server.query-metrics2-enabled", "false")); +} + public int[] getQueryMetricsPercentilesIntervals() { String[] dft = { "60", "300", "3600" }; return getOptionalIntArray("kylin.server.query-metrics-percentiles-intervals", dft); http://git-wip-us.apache.org/repos/asf/kylin/blob/b778f33a/core-common/src/main/java/org/apache/kylin/common/metrics/common/Metrics.java -- diff --git a/core-common/src/main/java/org/apache/kylin/common/metrics/common/Metrics.java b/core-common/src/main/java/org/apache/kylin/common/metrics/common/Metrics.java index dcda3cd..9bf8e7a 100644 --- a/core-common/src/main/java/org/apache/kylin/common/metrics/common/Metrics.java +++ b/core-common/src/main/java/org/apache/kylin/common/metrics/common/Metrics.java @@ -17,13 +17,13 @@ */ package org.apache.kylin.common.metrics.common; +import java.util.concurrent.TimeUnit; + import com.codahale.metrics.Counter; import com.codahale.metrics.Histogram; import com.codahale.metrics.Meter; import com.codahale.metrics.Timer; -import java.util.concurrent.TimeUnit; - /** * Generic Metics interface. */ @@ -35,7 +35,6 @@ public interface Metrics { public void close() throws Exception; /** - * * @param name starts a scope of a given name. Scopes is stored as thread-local variable. */ public void startStoredScope(String name); @@ -43,12 +42,14 @@ public interface Metrics { /** * Closes the stored scope of a given name. * Note that this must be called on the same thread as where the scope was started. + * * @param name */ public void endStoredScope(String name); /** * Create scope with given name and returns it. + * * @param name * @return */ @@ -56,6 +57,7 @@ public interface Metrics { /** * Close the given scope. + * * @param scope */ public void endScope(MetricsScope scope); @@ -64,6 +66,7 @@ public interface Metrics { /** * Increments a counter of the given name by 1. + * * @param name * @return */ @@ -71,6 +74,7 @@ public interface Metrics { /** * Increments a counter of the given name by "increment" + * * @param name * @param increment * @return @@ -79,6 +83,7 @@
[48/50] [abbrv] kylin git commit: Merge commit '727920b4d3642aaa3657d90b7f3dce7dcdd68fe2'
Merge commit '727920b4d3642aaa3657d90b7f3dce7dcdd68fe2' Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b50030da Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b50030da Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b50030da Branch: refs/heads/master Commit: b50030da4cd64d424a82e9d2b752c00917e8e681 Parents: 068baf6 727920b Author: Hongbin MaAuthored: Thu Aug 31 15:42:16 2017 +0800 Committer: Hongbin Ma Committed: Thu Aug 31 21:33:14 2017 +0800 -- assembly/pom.xml| 2 +- atopcalcite/pom.xml | 2 +- core-common/pom.xml | 2 +- .../org/apache/kylin/common/KylinConfig.java| 57 ++- .../apache/kylin/common/KylinConfigBase.java| 1 + .../org/apache/kylin/common/KylinConfigExt.java | 2 +- .../org/apache/kylin/common/KylinVersion.java | 4 +- .../org/apache/kylin/common/util/ClassUtil.java | 13 +- core-cube/pom.xml | 2 +- .../kylin/cube/gridtable/CubeCodeSystem.java| 4 + .../org/apache/kylin/gridtable/GTRecord.java| 8 + .../kylin/gridtable/GTSampleCodeSystem.java | 4 + .../apache/kylin/gridtable/IGTCodeSystem.java | 3 + core-dictionary/pom.xml | 2 +- core-job/pom.xml| 2 +- .../job/execution/DefaultChainedExecutable.java | 13 +- .../kylin/job/execution/ExecutableManager.java | 12 +- core-metadata/pom.xml | 2 +- .../measure/bitmap/BitmapCounterFactory.java| 2 + .../kylin/measure/bitmap/BitmapSerializer.java | 38 +- .../measure/bitmap/RoaringBitmapCounter.java| 10 + .../bitmap/RoaringBitmapCounterFactory.java | 5 + .../measure/percentile/PercentileCounter.java | 28 +- .../metadata/datatype/DataTypeSerializer.java | 13 + .../percentile/PercentileCounterTest.java | 46 --- core-storage/pom.xml| 2 +- .../gtrecord/GTCubeStorageQueryBase.java| 53 ++- .../gtrecord/SequentialCubeTupleIterator.java | 17 +- engine-mr/pom.xml | 2 +- .../engine/mr/common/AbstractHadoopJob.java | 128 +++--- .../kylin/engine/mr/common/BatchConstants.java | 1 + .../kylin/engine/mr/common/HadoopCmdOutput.java | 19 +- .../engine/mr/common/JobInfoConverter.java | 7 +- .../engine/mr/common/JobRelatedMetaUtil.java| 72 engine-spark/pom.xml| 2 +- .../engine/spark/KylinKryoRegistrator.java | 11 +- .../spark/SparkBatchCubingJobBuilder2.java | 5 + .../kylin/engine/spark/SparkCubingByLayer.java | 392 +++ .../kylin/engine/spark/SparkExecutable.java | 58 ++- .../spark/util/PercentileCounterSerializer.java | 55 +++ .../spark/util/PercentileSerializerTest.java| 67 .../acl/0928468a-9fab-4185-9a14-6f2e7c74823f| 27 ++ .../acl/2fbca32a-a33e-4b69-83dd-0bb8b1f8c53b| 27 ++ .../acl/2fbca32a-a33e-4b69-83dd-0bb8b1f8c91b| 24 ++ jdbc/pom.xml| 2 +- kylin-it/pom.xml| 18 +- .../job/ITDistributedSchedulerBaseTest.java | 2 + .../jdbc/ITJdbcSourceTableLoaderTest.java | 114 ++ .../source/jdbc/ITJdbcTableReaderTest.java | 109 ++ .../test/resources/query/sql_limit/query05.sql | 21 + pom.xml | 2 +- query/pom.xml | 2 +- .../query/adhoc/PushDownRunnerJdbcImpl.java | 2 - .../kylin/query/relnode/OLAPAggregateRel.java | 4 +- .../kylin/query/relnode/OLAPWindowRel.java | 2 - .../kylin/query/routing/RealizationChooser.java | 7 +- server-base/pom.xml | 2 +- .../kylin/rest/controller/CubeController.java | 49 ++- .../rest/controller2/CubeControllerV2.java | 19 +- .../apache/kylin/rest/service/AclService.java | 4 +- .../rest/service/AclTableMigrationTool.java | 13 +- .../apache/kylin/rest/service/JobService.java | 4 + .../apache/kylin/rest/service/QueryService.java | 4 +- .../kylin/rest/util/Log4jConfigListener.java| 1 + server/pom.xml | 2 +- server/src/main/resources/kylinSecurity.xml | 2 + .../kylin/rest/service/JobServiceTest.java | 33 ++ source-hive/pom.xml | 7 +- .../HiveColumnCardinalityUpdateJob.java | 3 +- .../apache/kylin/source/jdbc/JdbcExplorer.java | 6 +- source-kafka/pom.xml| 2 +- .../kylin/source/kafka/DateTimeParser.java | 62 +++ storage-hbase/pom.xml | 2 +- .../kylin/storage/hbase/HBaseConnection.java| 2 +-
[20/50] [abbrv] kylin git commit: KYLIN-2801 fix UT
KYLIN-2801 fix UT Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/68ae79a6 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/68ae79a6 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/68ae79a6 Branch: refs/heads/master Commit: 68ae79a61c03763c0e21f27fcb79b52db849d306 Parents: a3d42f1 Author: Hongbin MaAuthored: Tue Aug 22 15:32:01 2017 +0800 Committer: Hongbin Ma Committed: Tue Aug 22 16:00:14 2017 +0800 -- .../DefaultPartitionConditionBuilderTest.java | 33 +++- .../kylin/engine/mr/SortedColumnReaderTest.java | 17 -- 2 files changed, 41 insertions(+), 9 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/68ae79a6/core-metadata/src/test/java/org/apache/kylin/metadata/model/DefaultPartitionConditionBuilderTest.java -- diff --git a/core-metadata/src/test/java/org/apache/kylin/metadata/model/DefaultPartitionConditionBuilderTest.java b/core-metadata/src/test/java/org/apache/kylin/metadata/model/DefaultPartitionConditionBuilderTest.java index 1b610ca..907e0e5 100644 --- a/core-metadata/src/test/java/org/apache/kylin/metadata/model/DefaultPartitionConditionBuilderTest.java +++ b/core-metadata/src/test/java/org/apache/kylin/metadata/model/DefaultPartitionConditionBuilderTest.java @@ -19,11 +19,24 @@ package org.apache.kylin.metadata.model; import org.apache.kylin.common.util.DateFormat; +import org.apache.kylin.common.util.LocalFileMetadataTestCase; +import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; -public class DefaultPartitionConditionBuilderTest { +public class DefaultPartitionConditionBuilderTest extends LocalFileMetadataTestCase { + +@Before +public void setup() throws Exception { +this.createTestMetadata(); +} + +@After +public void after() throws Exception { +this.cleanupTestMetadata(); +} + private PartitionDesc.DefaultPartitionConditionBuilder partitionConditionBuilder; @Before @@ -38,8 +51,10 @@ public class DefaultPartitionConditionBuilderTest { partitionDesc.setPartitionDateColumnRef(col); partitionDesc.setPartitionDateColumn(col.getCanonicalName()); partitionDesc.setPartitionDateFormat("-MM-dd"); -String condition = partitionConditionBuilder.buildDateRangeCondition(partitionDesc, DateFormat.stringToMillis("2016-02-22"), DateFormat.stringToMillis("2016-02-23")); -Assert.assertEquals("UNKNOWN_ALIAS.DATE_COLUMN >= '2016-02-22' AND UNKNOWN_ALIAS.DATE_COLUMN < '2016-02-23'", condition); +String condition = partitionConditionBuilder.buildDateRangeCondition(partitionDesc, +DateFormat.stringToMillis("2016-02-22"), DateFormat.stringToMillis("2016-02-23")); +Assert.assertEquals("UNKNOWN_ALIAS.DATE_COLUMN >= '2016-02-22' AND UNKNOWN_ALIAS.DATE_COLUMN < '2016-02-23'", +condition); } @Test @@ -49,7 +64,8 @@ public class DefaultPartitionConditionBuilderTest { partitionDesc.setPartitionTimeColumnRef(col); partitionDesc.setPartitionTimeColumn(col.getCanonicalName()); partitionDesc.setPartitionTimeFormat("HH"); -String condition = partitionConditionBuilder.buildDateRangeCondition(partitionDesc, DateFormat.stringToMillis("2016-02-22 00:00:00"), DateFormat.stringToMillis("2016-02-23 01:00:00")); +String condition = partitionConditionBuilder.buildDateRangeCondition(partitionDesc, +DateFormat.stringToMillis("2016-02-22 00:00:00"), DateFormat.stringToMillis("2016-02-23 01:00:00")); Assert.assertEquals("UNKNOWN_ALIAS.HOUR_COLUMN >= '00' AND UNKNOWN_ALIAS.HOUR_COLUMN < '01'", condition); } @@ -64,8 +80,11 @@ public class DefaultPartitionConditionBuilderTest { partitionDesc.setPartitionTimeColumnRef(col2); partitionDesc.setPartitionTimeColumn(col2.getCanonicalName()); partitionDesc.setPartitionTimeFormat("H"); -String condition = partitionConditionBuilder.buildDateRangeCondition(partitionDesc, DateFormat.stringToMillis("2016-02-22 00:00:00"), DateFormat.stringToMillis("2016-02-23 01:00:00")); -Assert.assertEquals("((UNKNOWN_ALIAS.DATE_COLUMN = '2016-02-22' AND UNKNOWN_ALIAS.HOUR_COLUMN >= '0') OR (UNKNOWN_ALIAS.DATE_COLUMN > '2016-02-22')) AND ((UNKNOWN_ALIAS.DATE_COLUMN = '2016-02-23' AND UNKNOWN_ALIAS.HOUR_COLUMN < '1') OR (UNKNOWN_ALIAS.DATE_COLUMN < '2016-02-23'))", condition); +String condition = partitionConditionBuilder.buildDateRangeCondition(partitionDesc, +DateFormat.stringToMillis("2016-02-22 00:00:00"), DateFormat.stringToMillis("2016-02-23 01:00:00")); +
[08/50] [abbrv] kylin git commit: minor, update odbc driver version
minor, update odbc driver version Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/72a4f0f8 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/72a4f0f8 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/72a4f0f8 Branch: refs/heads/master Commit: 72a4f0f8c0b8b4ad440bf1f0b5ce35e687edd858 Parents: 7a9f74c Author: Yifan ZhangAuthored: Fri Aug 18 16:16:51 2017 +0800 Committer: Dong Li Committed: Fri Aug 18 16:24:43 2017 +0800 -- odbc/Installer(64bit)/Installer(64bit).isl | 2 +- odbc/Installer/Installer.isl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/72a4f0f8/odbc/Installer(64bit)/Installer(64bit).isl -- diff --git a/odbc/Installer(64bit)/Installer(64bit).isl b/odbc/Installer(64bit)/Installer(64bit).isl index 3393ac5..a3c96d5 100644 --- a/odbc/Installer(64bit)/Installer(64bit).isl +++ b/odbc/Installer(64bit)/Installer(64bit).isl @@ -4395,7 +4395,7 @@ UwBpAG4AZwBsAGUASQBtAGEAZwBlAAEARQB4AHAAcgBlAHMAcwA= PROGMSG_IIS_ROLLBACKWEBSERVICEEXTENSIONS##IDS_PROGMSG_IIS_ROLLBACKWEBSERVICEEXTENSIONS## ProductCode{C8B1B296-2F8B-423F-97D5-429C9B6D2AE6} ProductNameKylinODBCDriver (x64) - ProductVersion1.5. + ProductVersion2.1.0 ProgressType0install ProgressType1Installing ProgressType2installed http://git-wip-us.apache.org/repos/asf/kylin/blob/72a4f0f8/odbc/Installer/Installer.isl -- diff --git a/odbc/Installer/Installer.isl b/odbc/Installer/Installer.isl index a23656a..f6a82a6 100644 --- a/odbc/Installer/Installer.isl +++ b/odbc/Installer/Installer.isl @@ -4376,7 +4376,7 @@ UwBpAG4AZwBsAGUASQBtAGEAZwBlAAEARQB4AHAAcgBlAHMAcwA= PROGMSG_IIS_ROLLBACKWEBSERVICEEXTENSIONS##IDS_PROGMSG_IIS_ROLLBACKWEBSERVICEEXTENSIONS## ProductCode{A920FB5E-591B-4537-901D-7D030304} ProductNameKylinODBCDriver (x86) - ProductVersion1.5. + ProductVersion2.1.0 ProgressType0install ProgressType1Installing ProgressType2installed
[31/50] [abbrv] kylin git commit: KYLIN-2812, Fix the wrong destination database during load kafka topic (#2074)
KYLIN-2812, Fix the wrong destination database during load kafka topic (#2074) Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e2e0102c Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e2e0102c Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e2e0102c Branch: refs/heads/master Commit: e2e0102c4bac289c1e0d61226794030fd93cb145 Parents: ff4f634 Author: Billy(Yiming) LiuAuthored: Sun Aug 27 18:51:50 2017 +0800 Committer: GitHub Committed: Sun Aug 27 18:51:50 2017 +0800 -- .../rest/controller2/StreamingControllerV2.java | 3 +- .../controller/StreamingControllerTest.java | 52 2 files changed, 53 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/e2e0102c/server-base/src/main/java/org/apache/kylin/rest/controller2/StreamingControllerV2.java -- diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller2/StreamingControllerV2.java b/server-base/src/main/java/org/apache/kylin/rest/controller2/StreamingControllerV2.java index 54733ea..90f2a45 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/controller2/StreamingControllerV2.java +++ b/server-base/src/main/java/org/apache/kylin/rest/controller2/StreamingControllerV2.java @@ -247,10 +247,9 @@ public class StreamingControllerV2 extends BasicController { } if (null != desc) { -String[] dbTable = HadoopUtil.parseHiveTableName(desc.getName()); +String[] dbTable = HadoopUtil.parseHiveTableName(desc.getIdentity()); desc.setName(dbTable[1]); desc.setDatabase(dbTable[0]); -desc.getIdentity(); } return desc; } http://git-wip-us.apache.org/repos/asf/kylin/blob/e2e0102c/server-base/src/test/java/org/apache/kylin/rest/controller/StreamingControllerTest.java -- diff --git a/server-base/src/test/java/org/apache/kylin/rest/controller/StreamingControllerTest.java b/server-base/src/test/java/org/apache/kylin/rest/controller/StreamingControllerTest.java new file mode 100644 index 000..171d22e --- /dev/null +++ b/server-base/src/test/java/org/apache/kylin/rest/controller/StreamingControllerTest.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package org.apache.kylin.rest.controller; + +import org.apache.kylin.common.util.HadoopUtil; +import org.apache.kylin.common.util.JsonUtil; +import org.apache.kylin.metadata.model.TableDesc; +import org.junit.Assert; +import org.junit.Test; + +import java.io.IOException; + +public class StreamingControllerTest { + +@Test +public void testReadTableDesc() throws IOException { +String requestTableData = "{\"name\":\"my_table_name\",\"source_type\":1,\"columns\":[{\"id\":1,\"name\":" + + "\"amount\",\"datatype\":\"decimal\"},{\"id\":2,\"name\":\"category\",\"datatype\":\"varchar(256)\"}," + + "{\"id\":3,\"name\":\"order_time\",\"datatype\":\"timestamp\"},{\"id\":4,\"name\":\"device\"," + + "\"datatype\":\"varchar(256)\"},{\"id\":5,\"name\":\"qty\",\"datatype\":\"int\"},{\"id\":6,\"name\":" + + "\"user_id\",\"datatype\":\"varchar(256)\"},{\"id\":7,\"name\":\"user_age\",\"datatype\":\"int\"}," + + "{\"id\":8,\"name\":\"user_gender\",\"datatype\":\"varchar(256)\"},{\"id\":9,\"name\":\"currency\"," + + "\"datatype\":\"varchar(256)\"},{\"id\":10,\"name\":\"country\",\"datatype\":\"varchar(256)\"}," + + "{\"id\":11,\"name\":\"year_start\",\"datatype\":\"date\"},{\"id\":12,\"name\":\"quarter_start\"," + + "\"datatype\":\"date\"},{\"id\":13,\"name\":\"month_start\",\"datatype\":\"date\"},{\"id\":14," + + "\"name\":\"week_start\",\"datatype\":\"date\"},{\"id\":15,\"name\":\"day_start\",\"datatype\":" + +
[05/50] [abbrv] kylin git commit: KYLIN-2791 Fix bug in readLong in BytesUtil
KYLIN-2791 Fix bug in readLong in BytesUtil Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/fa7eb692 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/fa7eb692 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/fa7eb692 Branch: refs/heads/master Commit: fa7eb692b33218fba463f7839c00663bd83892f3 Parents: c487fe4 Author: auphyroc99 <454530...@qq.com> Authored: Wed Aug 16 13:55:40 2017 +0800 Committer: Hongbin MaCommitted: Wed Aug 16 14:13:36 2017 +0800 -- .../src/main/java/org/apache/kylin/common/util/BytesUtil.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/fa7eb692/core-common/src/main/java/org/apache/kylin/common/util/BytesUtil.java -- diff --git a/core-common/src/main/java/org/apache/kylin/common/util/BytesUtil.java b/core-common/src/main/java/org/apache/kylin/common/util/BytesUtil.java index 8cc559e..d41c6b4 100644 --- a/core-common/src/main/java/org/apache/kylin/common/util/BytesUtil.java +++ b/core-common/src/main/java/org/apache/kylin/common/util/BytesUtil.java @@ -133,7 +133,7 @@ public class BytesUtil { public static long readLong(ByteBuffer in) { long integer = 0; -int mask = 0xff; +long mask = 0xff; int shift = 0; for (int i = 0; i < 8; i++) { integer |= (in.get() << shift) & mask;
[41/50] [abbrv] kylin git commit: minor change getProject() in DataModelDesc
minor change getProject() in DataModelDesc Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/28ac7d9a Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/28ac7d9a Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/28ac7d9a Branch: refs/heads/master Commit: 28ac7d9aeeeb8818207a363231b1354652f9aa8d Parents: 487a854 Author: tttMelody <245915...@qq.com> Authored: Tue Aug 29 17:22:41 2017 +0800 Committer: Hongbin MaCommitted: Tue Aug 29 17:40:33 2017 +0800 -- .../java/org/apache/kylin/metadata/model/DataModelDesc.java | 4 ++-- .../org/apache/kylin/rest/controller/ModelController.java| 2 +- .../java/org/apache/kylin/rest/service/ModelService.java | 8 .../java/org/apache/kylin/rest/service/ModelServiceTest.java | 8 4 files changed, 11 insertions(+), 11 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/28ac7d9a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java -- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java index a5da833..b27d644 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java @@ -854,11 +854,11 @@ public class DataModelDesc extends RootPersistentEntity { } public String getProject() { -return ProjectManager.getInstance(KylinConfig.getInstanceFromEnv()).getProjectOfModel(this.getName()).getName(); +return ProjectManager.getInstance(getConfig()).getProjectOfModel(this.getName()).getName(); } public ProjectInstance getProjectInstance() { -return ProjectManager.getInstance(KylinConfig.getInstanceFromEnv()).getProjectOfModel(this.getName()); +return ProjectManager.getInstance(getConfig()).getProjectOfModel(this.getName()); } public static DataModelDesc getCopyOf(DataModelDesc orig) { http://git-wip-us.apache.org/repos/asf/kylin/blob/28ac7d9a/server-base/src/main/java/org/apache/kylin/rest/controller/ModelController.java -- diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/ModelController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/ModelController.java index 4226e87..c5bee9f 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/controller/ModelController.java +++ b/server-base/src/main/java/org/apache/kylin/rest/controller/ModelController.java @@ -131,7 +131,7 @@ public class ModelController extends BasicController { return modelRequest; } try { -modelDesc = modelService.updateModelAndDesc(modelDesc); +modelDesc = modelService.updateModelAndDesc(modelRequest.getProject(), modelDesc); } catch (AccessDeniedException accessDeniedException) { throw new ForbiddenException("You don't have right to update this model."); } catch (Exception e) { http://git-wip-us.apache.org/repos/asf/kylin/blob/28ac7d9a/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java -- diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java b/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java index 82632eb..42cf085 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java +++ b/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java @@ -143,8 +143,8 @@ public class ModelService extends BasicService { return createdDesc; } -public DataModelDesc updateModelAndDesc(DataModelDesc desc) throws IOException { -aclEvaluate.hasProjectWritePermission(desc.getProjectInstance()); +public DataModelDesc updateModelAndDesc(String project, DataModelDesc desc) throws IOException { +aclEvaluate.checkProjectWritePermission(project); getMetadataManager().updateDataModelDesc(desc); return desc; } @@ -428,7 +428,7 @@ public class ModelService extends BasicService { } public DataModelDesc updateModelToResourceStore(DataModelDesc modelDesc, String projectName) throws IOException { - aclEvaluate.hasProjectWritePermission(getProjectManager().getProject(projectName)); +aclEvaluate.checkProjectWritePermission(projectName); Message msg = MsgPicker.getMsg(); modelDesc.setDraft(false); @@ -445,7 +445,7 @@ public class ModelService extends BasicService {
[10/50] [abbrv] kylin git commit: KYLIN-2649 configuration of force limit on "select star" query
KYLIN-2649 configuration of force limit on "select star" query Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c4f125c4 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c4f125c4 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c4f125c4 Branch: refs/heads/master Commit: c4f125c4750ef8d0c754d7a00c2ad100c6999cf6 Parents: 6ec099b Author: Roger ShiAuthored: Sun Aug 20 13:41:16 2017 +0800 Committer: Roger Shi Committed: Sun Aug 20 14:06:28 2017 +0800 -- .../main/java/org/apache/kylin/common/KylinConfigBase.java| 6 ++ .../src/main/java/org/apache/kylin/query/util/QueryUtil.java | 7 +++ 2 files changed, 13 insertions(+) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/c4f125c4/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java -- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index 78abf22..df74f5e 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -936,6 +936,12 @@ abstract public class KylinConfigBase implements Serializable { return Integer.parseInt(getOptional("kylin.query.max-limit-pushdown", "1")); } +// Select star on large table is too slow for BI, add limit by default if missing +// https://issues.apache.org/jira/browse/KYLIN-2649 +public int getForceLimit() { +return Integer.parseInt(getOptional("kylin.query.force-limit", "-1")); +} + @Deprecated public int getScanThreshold() { return Integer.parseInt(getOptional("kylin.query.scan-threshold", "1000")); http://git-wip-us.apache.org/repos/asf/kylin/blob/c4f125c4/query/src/main/java/org/apache/kylin/query/util/QueryUtil.java -- diff --git a/query/src/main/java/org/apache/kylin/query/util/QueryUtil.java b/query/src/main/java/org/apache/kylin/query/util/QueryUtil.java index 29e2c26..4befd33 100644 --- a/query/src/main/java/org/apache/kylin/query/util/QueryUtil.java +++ b/query/src/main/java/org/apache/kylin/query/util/QueryUtil.java @@ -44,6 +44,7 @@ public class QueryUtil { public static String massageSql(String sql, String project, int limit, int offset) { sql = sql.trim(); sql = sql.replace("\r", " ").replace("\n", System.getProperty("line.separator")); +KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv(); while (sql.endsWith(";")) sql = sql.substring(0, sql.length() - 1); @@ -56,6 +57,12 @@ public class QueryUtil { sql += ("\nOFFSET " + offset); } +// https://issues.apache.org/jira/browse/KYLIN-2649 +if (kylinConfig.getForceLimit() > 0 && +!sql.toLowerCase().contains("limit") && sql.toLowerCase().contains("*")) { +sql += ("\nLIMIT " + kylinConfig.getForceLimit()); +} + // customizable SQL transformation if (queryTransformers == null) { initQueryTransformers();
[30/50] [abbrv] kylin git commit: KYLIN-2808, fix table not fould exception
KYLIN-2808, fix table not fould exception Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ff4f634f Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ff4f634f Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ff4f634f Branch: refs/heads/master Commit: ff4f634fe5ad5f96b1506a2ab368eb701f62c5dc Parents: 1fff421 Author: Cheng WangAuthored: Fri Aug 25 19:10:19 2017 +0800 Committer: æ Committed: Sat Aug 26 00:19:08 2017 +0800 -- .../apache/kylin/source/hive/HiveMRInput.java | 41 +--- 1 file changed, 26 insertions(+), 15 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/ff4f634f/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java -- diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java index 71a7152..096134c 100644 --- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java +++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java @@ -62,10 +62,12 @@ public class HiveMRInput implements IMRInput { @SuppressWarnings("unused") private static final Logger logger = LoggerFactory.getLogger(HiveMRInput.class); - + public static String getTableNameForHCat(TableDesc table) { String tableName = (table.isView()) ? table.getMaterializedName() : table.getName(); -return String.format("%s.%s", table.getDatabase(), tableName).toUpperCase(); +String database = (table.isView()) ? KylinConfig.getInstanceFromEnv().getHiveDatabaseForIntermediateTable() +: table.getDatabase(); +return String.format("%s.%s", database, tableName).toUpperCase(); } @Override @@ -139,17 +141,18 @@ public class HiveMRInput implements IMRInput { @Override public void addStepPhase1_CreateFlatTable(DefaultChainedExecutable jobFlow) { final String cubeName = CubingExecutableUtil.getCubeName(jobFlow.getParams()); -final KylinConfig cubeConfig = CubeManager.getInstance(KylinConfig.getInstanceFromEnv()).getCube(cubeName).getConfig(); +final KylinConfig cubeConfig = CubeManager.getInstance(KylinConfig.getInstanceFromEnv()).getCube(cubeName) +.getConfig(); final String hiveInitStatements = JoinedFlatTable.generateHiveInitStatements(flatTableDatabase); // create flat table first addStepPhase1_DoCreateFlatTable(jobFlow); - + // then count and redistribute if (cubeConfig.isHiveRedistributeEnabled()) { jobFlow.addTask(createRedistributeFlatHiveTableStep(hiveInitStatements, cubeName)); } - + // special for hive addStepPhase1_DoMaterializeLookupTable(jobFlow); } @@ -158,14 +161,14 @@ public class HiveMRInput implements IMRInput { final String cubeName = CubingExecutableUtil.getCubeName(jobFlow.getParams()); final String hiveInitStatements = JoinedFlatTable.generateHiveInitStatements(flatTableDatabase); final String jobWorkingDir = getJobWorkingDir(jobFlow); - + jobFlow.addTask(createFlatHiveTableStep(hiveInitStatements, jobWorkingDir, cubeName)); } protected void addStepPhase1_DoMaterializeLookupTable(DefaultChainedExecutable jobFlow) { final String hiveInitStatements = JoinedFlatTable.generateHiveInitStatements(flatTableDatabase); final String jobWorkingDir = getJobWorkingDir(jobFlow); - + AbstractExecutable task = createLookupHiveViewMaterializationStep(hiveInitStatements, jobWorkingDir); if (task != null) { jobFlow.addTask(task); @@ -186,7 +189,8 @@ public class HiveMRInput implements IMRInput { return step; } -private ShellExecutable createLookupHiveViewMaterializationStep(String hiveInitStatements, String jobWorkingDir) { +private ShellExecutable createLookupHiveViewMaterializationStep(String hiveInitStatements, +String jobWorkingDir) { ShellExecutable step = new ShellExecutable(); step.setName(ExecutableConstants.STEP_NAME_MATERIALIZE_HIVE_VIEW_IN_LOOKUP); @@ -215,21 +219,25 @@ public class HiveMRInput implements IMRInput { if (lookUpTableDesc.isView()) { StringBuilder createIntermediateTableHql = new StringBuilder(); createIntermediateTableHql.append("DROP TABLE IF EXISTS " + intermediate +
[03/50] [abbrv] kylin git commit: minor, fix getFromNode when SQL has limit/order by clause
minor, fix getFromNode when SQL has limit/order by clause Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2eca43c4 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2eca43c4 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2eca43c4 Branch: refs/heads/master Commit: 2eca43c40723cce0f12a56a1299894bbf5803d03 Parents: c8bab3d Author: Jiatao Tao <245915...@qq.com> Authored: Mon Aug 14 05:05:31 2017 -0500 Committer: GitHubCommitted: Mon Aug 14 05:05:31 2017 -0500 -- .../metadata/model/tool/CalciteParser.java | 20 ++-- .../apache/kylin/query/util/PushDownUtil.java | 4 .../kylin/query/util/PushDownUtilTest.java | 10 +- 3 files changed, 23 insertions(+), 11 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/2eca43c4/core-metadata/src/main/java/org/apache/kylin/metadata/model/tool/CalciteParser.java -- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/tool/CalciteParser.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/tool/CalciteParser.java index a48abae..01f0b19 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/tool/CalciteParser.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/tool/CalciteParser.java @@ -29,6 +29,7 @@ import org.apache.calcite.sql.SqlIdentifier; import org.apache.calcite.sql.SqlLiteral; import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.SqlNodeList; +import org.apache.calcite.sql.SqlOrderBy; import org.apache.calcite.sql.SqlSelect; import org.apache.calcite.sql.parser.SqlParseException; import org.apache.calcite.sql.parser.SqlParser; @@ -69,14 +70,21 @@ public class CalciteParser { } public static SqlNode getFromNode(String sql) { -//When the sql have limit clause, calcite will parse it as a SqlOrder Object. -sql = sql.split("LIMIT")[0]; -SqlNode fromNode = null; +SqlNode node = null; try { -fromNode = ((SqlSelect) (CalciteParser.parse(sql))).getFrom(); +node = CalciteParser.parse(sql); } catch (SqlParseException e) { -throw new RuntimeException("Failed to parse expression \'" + sql -+ "\', please make sure the expression is valid"); +throw new RuntimeException( +"Failed to parse expression \'" + sql + "\', please make sure the expression is valid"); +} +//When the sql have limit clause, calcite will parse it as a SqlOrder Object. +SqlNode fromNode = null; +if (node instanceof SqlOrderBy) { +SqlOrderBy orderBy = (SqlOrderBy) node; +fromNode = ((SqlSelect) orderBy.query).getFrom(); +} else { +SqlSelect sqlSelect = (SqlSelect) node; +fromNode = sqlSelect.getFrom(); } return fromNode; } http://git-wip-us.apache.org/repos/asf/kylin/blob/2eca43c4/query/src/main/java/org/apache/kylin/query/util/PushDownUtil.java -- diff --git a/query/src/main/java/org/apache/kylin/query/util/PushDownUtil.java b/query/src/main/java/org/apache/kylin/query/util/PushDownUtil.java index b2248f8..51352d2 100644 --- a/query/src/main/java/org/apache/kylin/query/util/PushDownUtil.java +++ b/query/src/main/java/org/apache/kylin/query/util/PushDownUtil.java @@ -112,6 +112,10 @@ public class PushDownUtil { FromTablesVisitor ftv = new FromTablesVisitor(); fromNode.accept(ftv); List tablesWithoutSchema = ftv.getTablesWithoutSchema(); +// sql do not need completion +if (tablesWithoutSchema.isEmpty()) { +return inputSql; +} List > tablesPos = new ArrayList<>(); for (SqlNode tables : tablesWithoutSchema) { http://git-wip-us.apache.org/repos/asf/kylin/blob/2eca43c4/query/src/test/java/org/apache/kylin/query/util/PushDownUtilTest.java -- diff --git a/query/src/test/java/org/apache/kylin/query/util/PushDownUtilTest.java b/query/src/test/java/org/apache/kylin/query/util/PushDownUtilTest.java index e2ccd9e..8ed58f7 100644 --- a/query/src/test/java/org/apache/kylin/query/util/PushDownUtilTest.java +++ b/query/src/test/java/org/apache/kylin/query/util/PushDownUtilTest.java @@ -25,12 +25,12 @@ import org.mockito.Mockito; public class PushDownUtilTest { @Test public void testSchemaCompletion() { -String sql1 = "SELECT a \n"+ +String sql1 = "SELECT a \n" + "FROM a.KYLIN_SALES as KYLIN_SALES\n" +
[17/50] [abbrv] kylin git commit: KYLIN-2575 code refactors
KYLIN-2575 code refactors Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/aed840f9 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/aed840f9 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/aed840f9 Branch: refs/heads/master Commit: aed840f9343e3e62a711517900c357a091e75619 Parents: 8906d13 Author: Hongbin MaAuthored: Mon Aug 21 11:51:15 2017 +0800 Committer: Hongbin Ma Committed: Mon Aug 21 22:55:59 2017 +0800 -- .../apache/kylin/common/KylinConfigBase.java| 18 +- .../common/util/CaseInsensitiveString.java | 63 +++ .../kylin/common/util/CliCommandExecutor.java | 5 +- .../org/apache/kylin/common/util/BasicTest.java | 8 +- .../cube/cuboid/DefaultCuboidScheduler.java | 18 +- .../org/apache/kylin/cube/model/CubeDesc.java | 92 ++ .../cube/model/TooManyCuboidException.java | 25 +++ .../org/apache/kylin/cube/CubeDescTest.java | 7 +- .../org/apache/kylin/job/JoinedFlatTable.java | 2 +- .../apache/kylin/metadata/model/ColumnDesc.java | 15 +- .../metadata/model/ComputedColumnDesc.java | 73 ++-- .../kylin/metadata/model/DataModelDesc.java | 21 ++- .../metadata/model/tool/CalciteParser.java | 102 ++- .../kylin/metadata/project/ProjectManager.java | 9 +- .../kylin/model/tool/CalciteParserTest.java | 16 -- .../gtrecord/GTCubeStorageQueryBase.java| 54 -- .../localmeta/cube_desc/ci_inner_join_cube.json | 46 +++-- .../localmeta/cube_desc/ci_left_join_cube.json | 46 +++-- .../cube_desc/ut_inner_join_cube_partial.json | 28 +-- .../model_desc/ci_inner_join_model.json | 52 -- .../model_desc/ci_left_join_model.json | 54 -- .../query/sql_computedcolumn/query01.sql| 2 +- .../query/sql_computedcolumn/query02.sql| 2 +- .../query/sql_computedcolumn/query03.sql| 2 +- .../query/sql_verifyCount/query01.sql.expected | 2 +- .../query/sql_verifyCount/query03.sql.expected | 2 +- .../query/sql_verifyCount/query04.sql.expected | 2 +- .../query/sql_verifyCount/query10.sql.expected | 2 +- .../query/sql_verifyCount/query11.sql.expected | 2 +- .../query/adhoc/PushDownRunnerJdbcImpl.java | 2 +- .../query/enumerator/LookupTableEnumerator.java | 2 +- .../kylin/query/relnode/ColumnRowType.java | 4 + .../kylin/query/relnode/OLAPProjectRel.java | 2 +- .../kylin/query/relnode/OLAPTableScan.java | 2 +- .../apache/kylin/query/schema/OLAPSchema.java | 12 +- .../kylin/query/schema/OLAPSchemaFactory.java | 2 +- .../apache/kylin/query/schema/OLAPTable.java| 22 ++- .../query/util/KeywordDefaultDirtyHack.java | 2 +- .../apache/kylin/query/util/PushDownUtil.java | 174 ++- .../org/apache/kylin/query/util/QueryUtil.java | 25 ++- .../kylin/query/util/PushDownUtilTest.java | 43 - .../apache/kylin/query/util/QueryUtilTest.java | 10 +- .../rest/controller2/ModelControllerV2.java | 34 +++- .../apache/kylin/rest/service/ModelService.java | 113 +++- .../apache/kylin/rest/service/QueryService.java | 75 45 files changed, 861 insertions(+), 433 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/aed840f9/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java -- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index df74f5e..0f5f3a9 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -605,7 +605,7 @@ abstract public class KylinConfigBase implements Serializable { public Map getKafkaConfigOverride() { return getPropertiesByPrefix("kylin.source.kafka.config-override."); } - + // // SOURCE.JDBC // @@ -613,23 +613,23 @@ abstract public class KylinConfigBase
[39/50] [abbrv] kylin git commit: Revert "KYLIN-2816 support hive unicode column name"
Revert "KYLIN-2816 support hive unicode column name" This reverts commit 3712464e5e09c66b4c17cdb0c19e420c61c13358. Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/487a854b Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/487a854b Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/487a854b Branch: refs/heads/master Commit: 487a854bfece7b9fc72b0118a40d51100eca8463 Parents: 81e9f57 Author: Hongbin MaAuthored: Mon Aug 28 19:19:24 2017 +0800 Committer: Hongbin Ma Committed: Tue Aug 29 10:36:10 2017 +0800 -- .../kylin/common/util/HiveCmdBuilder.java | 5 ++-- .../kylin/common/util/HiveCmdBuilderTest.java | 2 +- .../org/apache/kylin/job/JoinedFlatTable.java | 28 .../apache/kylin/source/hive/HiveMRInput.java | 2 +- 4 files changed, 9 insertions(+), 28 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/487a854b/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java -- diff --git a/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java b/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java index 191bbc0..2f6b9a0 100644 --- a/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java +++ b/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java @@ -66,12 +66,11 @@ public class HiveCmdBuilder { switch (clientMode) { case CLI: -// use single-quote to ignore the executing back-ticks surrounded column name in shell -buf.append("hive -e \'"); +buf.append("hive -e \""); for (String statement : statements) { buf.append(statement).append("\n"); } -buf.append("\'"); +buf.append("\""); buf.append(parseProps()); break; case BEELINE: http://git-wip-us.apache.org/repos/asf/kylin/blob/487a854b/core-common/src/test/java/org/apache/kylin/common/util/HiveCmdBuilderTest.java -- diff --git a/core-common/src/test/java/org/apache/kylin/common/util/HiveCmdBuilderTest.java b/core-common/src/test/java/org/apache/kylin/common/util/HiveCmdBuilderTest.java index d2938b5..d69d4d2 100644 --- a/core-common/src/test/java/org/apache/kylin/common/util/HiveCmdBuilderTest.java +++ b/core-common/src/test/java/org/apache/kylin/common/util/HiveCmdBuilderTest.java @@ -60,7 +60,7 @@ public class HiveCmdBuilderTest { hiveCmdBuilder.addStatement("SHOW\n TABLES;"); hiveCmdBuilder.setHiveConfProps(hiveProps); hiveCmdBuilder.overwriteHiveProps(hivePropsOverwrite); -assertEquals("hive -e \'USE default;\nDROP TABLE test;\nSHOW\n TABLES;\n\' --hiveconf hive.execution.engine=tez", hiveCmdBuilder.build()); +assertEquals("hive -e \"USE default;\nDROP TABLE test;\nSHOW\n TABLES;\n\" --hiveconf hive.execution.engine=tez", hiveCmdBuilder.build()); } @Test http://git-wip-us.apache.org/repos/asf/kylin/blob/487a854b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java -- diff --git a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java index 824c693..acb29e1 100644 --- a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java +++ b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java @@ -42,8 +42,6 @@ import org.apache.kylin.metadata.model.TblColRef; public class JoinedFlatTable { -public static final String BACKTICK = "`"; - public static String getTableDir(IJoinedFlatTableDesc flatDesc, String storageDfsDir) { return storageDfsDir + "/" + flatDesc.getTableName(); } @@ -77,7 +75,7 @@ public class JoinedFlatTable { ddl.append("ROW FORMAT DELIMITED FIELDS TERMINATED BY ','" + "\n"); } ddl.append("STORED AS " + format + "\n"); -ddl.append("LOCATION '\"'\"'" + getTableDir(flatDesc, storageDfsDir) + "'\"'\"';").append("\n"); +ddl.append("LOCATION '" + getTableDir(flatDesc, storageDfsDir) + "';").append("\n"); return ddl.toString(); } @@ -130,15 +128,10 @@ public class JoinedFlatTable { sql.append(","); } String colTotalName = String.format("%s.%s", col.getTableRef().getTableName(), col.getName()); -String expressionInSourceDB = col.getExpressionInSourceDB(); -if (expressionInSourceDB.contains(".")) { -// surround column name with back-tick, to support
[45/50] [abbrv] kylin git commit: KYLIN-2790 Add extending point for column family
KYLIN-2790 Add extending point for column family Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d62cb58b Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d62cb58b Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d62cb58b Branch: refs/heads/master Commit: d62cb58bde55766dbe87bbad29953dd1243c2014 Parents: 4782289 Author: auphyroc99 <454530...@qq.com> Authored: Wed Aug 16 10:34:02 2017 +0800 Committer: Roger ShiCommitted: Wed Aug 30 18:13:52 2017 +0800 -- .../apache/kylin/common/KylinConfigBase.java| 4 .../org/apache/kylin/cube/model/CubeDesc.java | 18 ++-- .../kylin/cube/model/HBaseMappingDesc.java | 18 .../org/apache/kylin/gridtable/GTRecord.java| 22 4 files changed, 60 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/d62cb58b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java -- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index a113327..47806a5 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -286,6 +286,10 @@ abstract public class KylinConfigBase implements Serializable { "org.apache.kylin.storage.hbase.util.ZookeeperDistributedLock$Factory"); return (DistributedLockFactory) ClassUtil.newInstance(clsName); } + +public String getHBaseMappingAdapter() { +return getOptional("kylin.metadata.hbasemapping-adapter"); +} // // DICTIONARY & SNAPSHOT http://git-wip-us.apache.org/repos/asf/kylin/blob/d62cb58b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java -- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java index 0e22587..d263e1f 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java @@ -22,6 +22,7 @@ import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; import static com.google.common.base.Preconditions.checkState; +import java.lang.reflect.Method; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.util.ArrayList; @@ -603,8 +604,21 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware { validateAggregationGroups(); // check if aggregation group is valid validateAggregationGroupsCombination(); -if (hbaseMapping != null) { -hbaseMapping.init(this); +String hbaseMappingAdapterName = config.getHBaseMappingAdapter(); + +if (hbaseMappingAdapterName != null) { +try { +Class hbaseMappingAdapterClass = Class.forName(hbaseMappingAdapterName); +Method initMethod = hbaseMappingAdapterClass.getMethod("initHBaseMapping", CubeDesc.class); +initMethod.invoke(null, this); +} catch (Exception e) { +logger.error("Wrong configuration for kylin.metadata.hbasemapping-adapter: class " ++ hbaseMappingAdapterName + " not found. "); +} +} else { +if (hbaseMapping != null) { +hbaseMapping.init(this); +} } initMeasureReferenceToColumnFamily(); http://git-wip-us.apache.org/repos/asf/kylin/blob/d62cb58b/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseMappingDesc.java -- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseMappingDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseMappingDesc.java index d1e5829..77b4b14 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseMappingDesc.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseMappingDesc.java @@ -87,6 +87,24 @@ public class HBaseMappingDesc implements java.io.Serializable { } } +public void initAsSeparatedColumns(CubeDesc cubeDesc) { +cubeRef = cubeDesc; + +int cfNum = cubeDesc.getMeasures().size(); +columnFamily = new HBaseColumnFamilyDesc[cfNum]; + +for (int i = 0; i < cfNum; i++) { +
[34/50] [abbrv] kylin git commit: minor, avoid buffer size overflow
minor, avoid buffer size overflow Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/fad75c93 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/fad75c93 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/fad75c93 Branch: refs/heads/master Commit: fad75c93f42d252dd1ee2ec7b03df716c15fb2f4 Parents: 3200d6a Author: Hongbin MaAuthored: Mon Aug 28 12:18:23 2017 +0800 Committer: Hongbin Ma Committed: Mon Aug 28 12:25:12 2017 +0800 -- .../kylin/metadata/filter/TupleFilterSerializer.java | 13 + 1 file changed, 9 insertions(+), 4 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/fad75c93/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java -- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java index 63153ef..d434f0b 100755 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java @@ -69,8 +69,11 @@ public class TupleFilterSerializer { internalSerialize(rootFilter, decorator, buffer, cs); break; } catch (BufferOverflowException e) { -logger.info("Buffer size {} cannot hold the filter, resizing to 4 times", bufferSize); -bufferSize *= 4; +if (bufferSize == (1 << 30)) +throw e; + +logger.info("Buffer size {} cannot hold the filter, resizing to 2 times", bufferSize); +bufferSize = bufferSize << 1; } } byte[] result = new byte[buffer.position()]; @@ -78,7 +81,8 @@ public class TupleFilterSerializer { return result; } -private static void internalSerialize(TupleFilter filter, Decorator decorator, ByteBuffer buffer, IFilterCodeSystem cs) { +private static void internalSerialize(TupleFilter filter, Decorator decorator, ByteBuffer buffer, +IFilterCodeSystem cs) { if (decorator != null) { // give decorator a chance to manipulate the output filter filter = decorator.onSerialize(filter); } @@ -203,7 +207,8 @@ public class TupleFilterSerializer { if (extendedTupleFilters.containsKey(op)) { try { filter = (TupleFilter) extendedTupleFilters.get(op).getConstructor().newInstance(); -} catch (InstantiationException | IllegalAccessException | InvocationTargetException | NoSuchMethodException e) { +} catch (InstantiationException | IllegalAccessException | InvocationTargetException +| NoSuchMethodException e) { throw new RuntimeException(e); } } else {
[26/50] [abbrv] kylin git commit: minor, fix flanky test query
minor, fix flanky test query Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/cd9a5152 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/cd9a5152 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/cd9a5152 Branch: refs/heads/master Commit: cd9a515286eb7ba0553339cfb98d86ea10377a90 Parents: 8e07884 Author: Hongbin MaAuthored: Wed Aug 23 14:05:47 2017 +0800 Committer: Hongbin Ma Committed: Thu Aug 24 10:30:24 2017 +0800 -- kylin-it/src/test/resources/query/sql/query98.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/cd9a5152/kylin-it/src/test/resources/query/sql/query98.sql -- diff --git a/kylin-it/src/test/resources/query/sql/query98.sql b/kylin-it/src/test/resources/query/sql/query98.sql index d73a4cb..81736d9 100644 --- a/kylin-it/src/test/resources/query/sql/query98.sql +++ b/kylin-it/src/test/resources/query/sql/query98.sql @@ -16,6 +16,6 @@ -- limitations under the License. -- -select cal_dt, sum(price) as x,leaf_categ_id as GMV +select cal_dt,leaf_categ_id as x, sum(price) as GMV from test_kylin_fact group by leaf_categ_id, cal_dt
[14/50] [abbrv] kylin git commit: minor, more checks on computed column
minor, more checks on computed column Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/788c7313 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/788c7313 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/788c7313 Branch: refs/heads/master Commit: 788c731337cf76aab14ac388fa06782943cdf711 Parents: 6e28a02 Author: Hongbin MaAuthored: Mon Aug 21 22:10:36 2017 +0800 Committer: Hongbin Ma Committed: Mon Aug 21 22:55:59 2017 +0800 -- .../kylin/metadata/model/ComputedColumnDesc.java | 17 - .../apache/kylin/metadata/model/DataModelDesc.java | 2 +- .../resources/query/sql_computedcolumn/query04.sql | 1 + .../query/sql_computedcolumn/query04.sql.compare | 2 ++ 4 files changed, 20 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/788c7313/core-metadata/src/main/java/org/apache/kylin/metadata/model/ComputedColumnDesc.java -- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ComputedColumnDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ComputedColumnDesc.java index 9911fd2..bd785c8 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ComputedColumnDesc.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ComputedColumnDesc.java @@ -18,6 +18,7 @@ package org.apache.kylin.metadata.model; import java.io.Serializable; +import java.util.Map; import java.util.Set; import org.apache.calcite.sql.SqlAsOperator; @@ -55,7 +56,9 @@ public class ComputedColumnDesc implements Serializable { @JsonProperty private String comment; -public void init(Set aliasSet, String rootFactTableName) { +public void init(Map aliasMap, String rootFactTableName) { +Set aliasSet = aliasMap.keySet(); + Preconditions.checkNotNull(tableIdentity, "tableIdentity is null"); Preconditions.checkNotNull(columnName, "columnName is null"); Preconditions.checkNotNull(expression, "expression is null"); @@ -81,6 +84,18 @@ public class ComputedColumnDesc implements Serializable { throw new IllegalArgumentException("Computed column has to be defined on fact table"); } +for (TableRef tableRef : aliasMap.values()) { +if (!rootFactTableName.equals(tableRef.getAlias())) { +for (TblColRef tblColRef : tableRef.getColumns()) { +if (this.columnName.equals(tblColRef.getName())) { +throw new IllegalArgumentException( +"Computed column name " + columnName + " is already found on table " ++ tableRef.getTableIdentity() + ", use a different computed column name"); +} +} +} +} + if ("true".equals(System.getProperty("needCheckCC"))) { //conditional execute this because of the calcite dependency is to available every where try { simpleParserCheck(expression, aliasSet); http://git-wip-us.apache.org/repos/asf/kylin/blob/788c7313/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java -- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java index fd609b8..a5da833 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java @@ -501,7 +501,7 @@ public class DataModelDesc extends RootPersistentEntity { for (ComputedColumnDesc newCC : this.computedColumnDescs) { -newCC.init(aliasMap.keySet(), rootFactTableRef.getAlias()); +newCC.init(aliasMap, rootFactTableRef.getAlias()); final String newCCFullName = newCC.getFullName(); final String newCCColumnName = newCC.getColumnName(); http://git-wip-us.apache.org/repos/asf/kylin/blob/788c7313/kylin-it/src/test/resources/query/sql_computedcolumn/query04.sql -- diff --git a/kylin-it/src/test/resources/query/sql_computedcolumn/query04.sql b/kylin-it/src/test/resources/query/sql_computedcolumn/query04.sql new file mode 100644 index 000..780de5d --- /dev/null +++ b/kylin-it/src/test/resources/query/sql_computedcolumn/query04.sql @@ -0,0 +1 @@ +select count(*) as a , sum (F.DEAL_AMOUNT) as b from test_kylin_fact f inner join test_account a on f.seller_id =
[47/50] [abbrv] kylin git commit: minor, refactor pushdown code
minor, refactor pushdown code Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/068baf6b Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/068baf6b Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/068baf6b Branch: refs/heads/master Commit: 068baf6b3ec13fe6e68e9a41593dcd139b4a8597 Parents: a4aeb77 Author: Li YangAuthored: Wed Aug 30 21:18:19 2017 +0800 Committer: Roger Shi Committed: Thu Aug 31 10:37:32 2017 +0800 -- .../kylin/common/debug/BackdoorToggles.java | 19 +++- .../source/adhocquery/IPushDownRunner.java | 2 +- .../org/apache/kylin/query/KylinTestBase.java | 11 +- .../query/adhoc/PushDownRunnerJdbcImpl.java | 5 +- .../apache/kylin/query/relnode/OLAPContext.java | 11 +- .../apache/kylin/query/relnode/OLAPJoinRel.java | 1 + .../relnode/OLAPToEnumerableConverter.java | 61 ++- .../routing/RoutingIndicatorException.java | 35 +++ .../apache/kylin/query/util/PushDownUtil.java | 104 ++- .../apache/kylin/rest/service/QueryService.java | 37 --- 10 files changed, 145 insertions(+), 141 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/068baf6b/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java -- diff --git a/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java b/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java index 8cb48b6..88c7c16 100644 --- a/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java +++ b/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java @@ -40,6 +40,14 @@ public class BackdoorToggles { _backdoorToggles.set(toggles); } +public static void addToggle(String key, String value) { +Map map = _backdoorToggles.get(); +if (map == null) { +setToggles(Maps. newHashMap()); +} +_backdoorToggles.get().put(key, value); +} + public static void addToggles(Map toggles) { Map map = _backdoorToggles.get(); if (map == null) { @@ -47,6 +55,15 @@ public class BackdoorToggles { } _backdoorToggles.get().putAll(toggles); } + +// try avoid using this generic method +public static String getToggle(String key) { +Map map = _backdoorToggles.get(); +if (map == null) +return null; + +return map.get(key); +} public static String getCoprocessorBehavior() { return getString(DEBUG_TOGGLE_COPROCESSOR_BEHAVIOR); @@ -109,7 +126,7 @@ public class BackdoorToggles { public static boolean getPrepareOnly() { return getBoolean(DEBUG_TOGGLE_PREPARE_ONLY); } - + private static String getString(String key) { Map toggles = _backdoorToggles.get(); if (toggles == null) { http://git-wip-us.apache.org/repos/asf/kylin/blob/068baf6b/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/IPushDownRunner.java -- diff --git a/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/IPushDownRunner.java b/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/IPushDownRunner.java index 0336bfb..9983f5c 100644 --- a/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/IPushDownRunner.java +++ b/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/IPushDownRunner.java @@ -47,6 +47,6 @@ public interface IPushDownRunner { * * @throws Exception if running pushdown fails */ -boolean executeUpdate(String sql) throws Exception; +void executeUpdate(String sql) throws Exception; } http://git-wip-us.apache.org/repos/asf/kylin/blob/068baf6b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java -- diff --git a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java index 8f143c7..d3db995 100644 --- a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java +++ b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java @@ -70,7 +70,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Lists; import com.google.common.io.Files; /** @@ -261,14 +260,12 @@ public class KylinTestBase { return output(resultSet, needDisplay); } catch (SQLException
[11/50] [abbrv] kylin git commit: minor, fix odbc version
minor, fix odbc version Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1c8de93c Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1c8de93c Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1c8de93c Branch: refs/heads/master Commit: 1c8de93c33cdcb4b032409d0fea37885014b3cab Parents: c4f125c Author: lidongsjtuAuthored: Mon Aug 21 12:55:05 2017 +0800 Committer: Yifan Zhang Committed: Mon Aug 21 13:48:25 2017 +0800 -- odbc/Driver/GODBC.RC | 480 ++-- odbc/Driver/resource.h | 105 +- 2 files changed, 292 insertions(+), 293 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/1c8de93c/odbc/Driver/GODBC.RC -- diff --git a/odbc/Driver/GODBC.RC b/odbc/Driver/GODBC.RC index 9e0f4c3..73b17f9 100644 --- a/odbc/Driver/GODBC.RC +++ b/odbc/Driver/GODBC.RC @@ -1,240 +1,240 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Microsoft Visual C++ generated resource script. -// -#include "resource.h" - -#define APSTUDIO_READONLY_SYMBOLS -/ -// -// Generated from the TEXTINCLUDE 2 resource. -// -#include "afxres.h" - -/ -#undef APSTUDIO_READONLY_SYMBOLS - -/ -// Chinese (Simplified, PRC) resources - -#if !defined(AFX_RESOURCE_DLL) || defined(AFX_TARG_CHS) -LANGUAGE LANG_CHINESE, SUBLANG_CHINESE_SIMPLIFIED -#pragma code_page(936) - -/ -// -// Bitmap -// - -IDB_BITMAP1 BITMAP "banner.bmp" -#endif// Chinese (Simplified, PRC) resources -/ - - -/ -// English (United States) resources - -#if !defined(AFX_RESOURCE_DLL) || defined(AFX_TARG_ENU) -LANGUAGE LANG_ENGLISH, SUBLANG_ENGLISH_US -#pragma code_page(1252) - -#ifdef APSTUDIO_INVOKED -/ -// -// TEXTINCLUDE -// - -1 TEXTINCLUDE -BEGIN -"resource.h\0" -END - -2 TEXTINCLUDE -BEGIN -"#include ""afxres.h""\r\n" -"\0" -END - -3 TEXTINCLUDE -BEGIN -"\r\n" -"\0" -END - -#endif// APSTUDIO_INVOKED - - -/ -// -// Version -// - -VS_VERSION_INFO VERSIONINFO - FILEVERSION 1,0,0,1 - PRODUCTVERSION 1,0,0,1 - FILEFLAGSMASK 0x3fL -#ifdef _DEBUG - FILEFLAGS 0x1L -#else - FILEFLAGS 0x0L -#endif - FILEOS 0x40004L - FILETYPE 0x2L - FILESUBTYPE 0x0L -BEGIN -BLOCK "StringFileInfo" -BEGIN -BLOCK "040904b0" -BEGIN -VALUE "CompanyName", "kylinolap" -VALUE "FileDescription", "Kylin ODBC Driver" -VALUE "FileVersion", "1, 0, 0, 1" -VALUE "InternalName", "KylinODBC" -VALUE "LegalCopyright", "Copyright ?2014" -VALUE "OriginalFilename", "driver.dll" -VALUE "ProductName", "Kylin ODBC Driver" -VALUE "ProductVersion", "1, 0, 0, 1" -END -END -BLOCK "VarFileInfo" -BEGIN -VALUE "Translation", 0x409, 1200 -END -END - - -/ -// -// Dialog -// - -IDD_DSN_CFG1 DIALOGEX 0, 0, 265, 210 -STYLE DS_SETFONT | DS_MODALFRAME | DS_CENTER | WS_POPUP | WS_CAPTION -CAPTION "Apache Kylin Driver Connection" -FONT 8, "MS Sans Serif", 0, 0, 0x0 -BEGIN -PUSHBUTTON "Cancel",IDCANCEL,137,182,50,14 -LTEXT "Server Host",IDC_STATIC,44,74,38,8 -LTEXT "Port",IDC_STATIC,68,90,14,8 -EDITTEXTIDC_SERVER,91,72,110,12,ES_AUTOHSCROLL -EDITTEXT
[15/50] [abbrv] kylin git commit: KYLIN-2575 code refactor
KYLIN-2575 code refactor Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6e28a020 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6e28a020 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6e28a020 Branch: refs/heads/master Commit: 6e28a020309796289d892a704f6a7f62cc7512c9 Parents: aed840f Author: Hongbin MaAuthored: Mon Aug 21 20:35:15 2017 +0800 Committer: Hongbin Ma Committed: Mon Aug 21 22:55:59 2017 +0800 -- .../kylin/measure/MeasureTypeFactory.java | 24 ++- .../metadata/model/ComputedColumnDesc.java | 37 +++- .../kylin/metadata/model/DataModelDesc.java | 19 ++ .../metadata/model/tool/CalciteParser.java | 43 ++-- .../adhocquery/HivePushDownConverter.java | 2 +- .../source/adhocquery/IPushDownConverter.java | 2 +- .../kylin/model/tool/CalciteParserTest.java | 6 +- .../apache/kylin/query/util/PushDownUtil.java | 119 ++- .../kylin/query/util/PushDownUtilTest.java | 207 +-- .../apache/kylin/rest/service/ModelService.java | 3 +- 10 files changed, 150 insertions(+), 312 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/6e28a020/core-metadata/src/main/java/org/apache/kylin/measure/MeasureTypeFactory.java -- diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureTypeFactory.java b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureTypeFactory.java index 7f3a5f1..be5e12c 100644 --- a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureTypeFactory.java +++ b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureTypeFactory.java @@ -121,7 +121,8 @@ abstract public class MeasureTypeFactory { logger.info("Checking custom measure types from kylin config: " + customFactory); factoryInsts.add((MeasureTypeFactory>) Class.forName(customFactory).newInstance()); } catch (Exception e) { -throw new IllegalArgumentException("Unrecognized MeasureTypeFactory classname: " + customFactory, e); +throw new IllegalArgumentException("Unrecognized MeasureTypeFactory classname: " + customFactory, +e); } } } catch (KylinConfigCannotInitException e) { @@ -132,10 +133,12 @@ abstract public class MeasureTypeFactory { for (MeasureTypeFactory> factory : factoryInsts) { String funcName = factory.getAggrFunctionName(); if (funcName.equals(funcName.toUpperCase()) == false) -throw new IllegalArgumentException("Aggregation function name '" + funcName + "' must be in upper case"); +throw new IllegalArgumentException( +"Aggregation function name '" + funcName + "' must be in upper case"); String dataTypeName = factory.getAggrDataTypeName(); if (dataTypeName.equals(dataTypeName.toLowerCase()) == false) -throw new IllegalArgumentException("Aggregation data type name '" + dataTypeName + "' must be in lower case"); +throw new IllegalArgumentException( +"Aggregation data type name '" + dataTypeName + "' must be in lower case"); Class extends DataTypeSerializer>> serializer = factory.getAggrDataTypeSerializer(); logger.info("registering " + funcName + "(" + dataTypeName + "), " + factory.getClass()); @@ -153,7 +156,8 @@ abstract public class MeasureTypeFactory { } private static void registerUDAF(MeasureTypeFactory> factory) { -MeasureType> type = factory.createMeasureType(factory.getAggrFunctionName(), DataType.getType(factory.getAggrDataTypeName())); +MeasureType> type = factory.createMeasureType(factory.getAggrFunctionName(), +DataType.getType(factory.getAggrDataTypeName())); Map > udafs = type.getRewriteCalciteAggrFunctions(); if (type.needRewrite() == false || udafs == null) return; @@ -164,7 +168,8 @@ abstract public class MeasureTypeFactory { continue; //
[37/50] [abbrv] kylin git commit: minor, fix some error messages.
minor, fix some error messages. Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/480592d4 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/480592d4 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/480592d4 Branch: refs/heads/master Commit: 480592d49b8479d4cbba74e889eb5e256d9b2ef8 Parents: 74c904e Author: Roger ShiAuthored: Mon Aug 28 15:41:59 2017 +0800 Committer: Roger Shi Committed: Mon Aug 28 15:47:38 2017 +0800 -- server-base/src/main/java/org/apache/kylin/rest/msg/CnMessage.java | 2 +- server-base/src/main/java/org/apache/kylin/rest/msg/Message.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/480592d4/server-base/src/main/java/org/apache/kylin/rest/msg/CnMessage.java -- diff --git a/server-base/src/main/java/org/apache/kylin/rest/msg/CnMessage.java b/server-base/src/main/java/org/apache/kylin/rest/msg/CnMessage.java index ece4b83..2b1bf8e 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/msg/CnMessage.java +++ b/server-base/src/main/java/org/apache/kylin/rest/msg/CnMessage.java @@ -187,7 +187,7 @@ public class CnMessage extends Message { } public String getUPDATE_MODEL_KEY_FIELD() { -return "已使ç¨ç维度å度éä¸ç°æçè¿æ¥æ ä¸è½è¢«ä¿®æ¹"; +return "ç±äºç»´åº¦ã度éæè è¿æ¥å ³ç³»è¢«ä¿®æ¹å¯¼è´ä¸åå¨çcubeå®ä¹ä¸ä¸è´ï¼å èå½å模åæ æ³ä¿åã"; } public String getBROKEN_MODEL_DESC() { http://git-wip-us.apache.org/repos/asf/kylin/blob/480592d4/server-base/src/main/java/org/apache/kylin/rest/msg/Message.java -- diff --git a/server-base/src/main/java/org/apache/kylin/rest/msg/Message.java b/server-base/src/main/java/org/apache/kylin/rest/msg/Message.java index 8137c34..146a0a5 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/msg/Message.java +++ b/server-base/src/main/java/org/apache/kylin/rest/msg/Message.java @@ -187,7 +187,7 @@ public class Message { } public String getUPDATE_MODEL_KEY_FIELD() { -return "Dimensions and measures in use and existing join tree cannot be modified."; +return "Model cannot save because there are dimensions, measures or join relations modified to be inconsistent with existing cube."; } public String getBROKEN_MODEL_DESC() {
[32/50] [abbrv] kylin git commit: KYLIN-2816 support hive unicode column name
KYLIN-2816 support hive unicode column name Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/28a07ebd Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/28a07ebd Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/28a07ebd Branch: refs/heads/master Commit: 28a07ebd3983c325c64c0642e7dedbf32c284f39 Parents: e2e0102 Author: Billy(Yiming) LiuAuthored: Sun Aug 27 18:55:03 2017 +0800 Committer: GitHub Committed: Sun Aug 27 18:55:03 2017 +0800 -- .../kylin/common/util/HiveCmdBuilder.java | 5 ++-- .../kylin/common/util/HiveCmdBuilderTest.java | 2 +- .../org/apache/kylin/job/JoinedFlatTable.java | 28 .../apache/kylin/source/hive/HiveMRInput.java | 2 +- 4 files changed, 28 insertions(+), 9 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/28a07ebd/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java -- diff --git a/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java b/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java index 2f6b9a0..191bbc0 100644 --- a/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java +++ b/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java @@ -66,11 +66,12 @@ public class HiveCmdBuilder { switch (clientMode) { case CLI: -buf.append("hive -e \""); +// use single-quote to ignore the executing back-ticks surrounded column name in shell +buf.append("hive -e \'"); for (String statement : statements) { buf.append(statement).append("\n"); } -buf.append("\""); +buf.append("\'"); buf.append(parseProps()); break; case BEELINE: http://git-wip-us.apache.org/repos/asf/kylin/blob/28a07ebd/core-common/src/test/java/org/apache/kylin/common/util/HiveCmdBuilderTest.java -- diff --git a/core-common/src/test/java/org/apache/kylin/common/util/HiveCmdBuilderTest.java b/core-common/src/test/java/org/apache/kylin/common/util/HiveCmdBuilderTest.java index d69d4d2..d2938b5 100644 --- a/core-common/src/test/java/org/apache/kylin/common/util/HiveCmdBuilderTest.java +++ b/core-common/src/test/java/org/apache/kylin/common/util/HiveCmdBuilderTest.java @@ -60,7 +60,7 @@ public class HiveCmdBuilderTest { hiveCmdBuilder.addStatement("SHOW\n TABLES;"); hiveCmdBuilder.setHiveConfProps(hiveProps); hiveCmdBuilder.overwriteHiveProps(hivePropsOverwrite); -assertEquals("hive -e \"USE default;\nDROP TABLE test;\nSHOW\n TABLES;\n\" --hiveconf hive.execution.engine=tez", hiveCmdBuilder.build()); +assertEquals("hive -e \'USE default;\nDROP TABLE test;\nSHOW\n TABLES;\n\' --hiveconf hive.execution.engine=tez", hiveCmdBuilder.build()); } @Test http://git-wip-us.apache.org/repos/asf/kylin/blob/28a07ebd/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java -- diff --git a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java index acb29e1..824c693 100644 --- a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java +++ b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java @@ -42,6 +42,8 @@ import org.apache.kylin.metadata.model.TblColRef; public class JoinedFlatTable { +public static final String BACKTICK = "`"; + public static String getTableDir(IJoinedFlatTableDesc flatDesc, String storageDfsDir) { return storageDfsDir + "/" + flatDesc.getTableName(); } @@ -75,7 +77,7 @@ public class JoinedFlatTable { ddl.append("ROW FORMAT DELIMITED FIELDS TERMINATED BY ','" + "\n"); } ddl.append("STORED AS " + format + "\n"); -ddl.append("LOCATION '" + getTableDir(flatDesc, storageDfsDir) + "';").append("\n"); +ddl.append("LOCATION '\"'\"'" + getTableDir(flatDesc, storageDfsDir) + "'\"'\"';").append("\n"); return ddl.toString(); } @@ -128,10 +130,15 @@ public class JoinedFlatTable { sql.append(","); } String colTotalName = String.format("%s.%s", col.getTableRef().getTableName(), col.getName()); +String expressionInSourceDB = col.getExpressionInSourceDB(); +if (expressionInSourceDB.contains(".")) { +// surround column name with back-tick, to support unicode column name +expressionInSourceDB =
[09/50] [abbrv] kylin git commit: minor, fix display of hbase storage size
minor, fix display of hbase storage size Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6ec099b3 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6ec099b3 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6ec099b3 Branch: refs/heads/master Commit: 6ec099b31b448189930187612aa5be782f910d51 Parents: 72a4f0f Author: Cheng WangAuthored: Fri Aug 18 20:35:25 2017 +0800 Committer: Roger Shi Committed: Sat Aug 19 21:09:59 2017 +0800 -- .../apache/kylin/rest/service/CubeService.java | 48 +--- 1 file changed, 41 insertions(+), 7 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/6ec099b3/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java -- diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java index 16e94c7..f8ca38d 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java +++ b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java @@ -24,7 +24,6 @@ import java.util.Collections; import java.util.Date; import java.util.EnumSet; import java.util.List; -import java.util.WeakHashMap; import org.apache.commons.lang.StringUtils; import org.apache.kylin.common.KylinConfig; @@ -41,6 +40,7 @@ import org.apache.kylin.engine.mr.CubingJob; import org.apache.kylin.job.exception.JobException; import org.apache.kylin.job.execution.DefaultChainedExecutable; import org.apache.kylin.job.execution.ExecutableState; +import org.apache.kylin.metadata.cachesync.Broadcaster; import org.apache.kylin.metadata.draft.Draft; import org.apache.kylin.metadata.model.DataModelDesc; import org.apache.kylin.metadata.model.SegmentStatusEnum; @@ -52,6 +52,7 @@ import org.apache.kylin.metadata.realization.RealizationType; import org.apache.kylin.rest.constant.Constant; import org.apache.kylin.rest.exception.BadRequestException; import org.apache.kylin.rest.exception.ForbiddenException; +import org.apache.kylin.rest.exception.InternalErrorException; import org.apache.kylin.rest.msg.Message; import org.apache.kylin.rest.msg.MsgPicker; import org.apache.kylin.rest.request.MetricsRequest; @@ -61,6 +62,7 @@ import org.apache.kylin.rest.security.AclPermission; import org.apache.kylin.rest.util.AclEvaluate; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.InitializingBean; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.security.access.AccessDeniedException; @@ -68,6 +70,8 @@ import org.springframework.security.access.prepost.PreAuthorize; import org.springframework.security.core.context.SecurityContextHolder; import org.springframework.stereotype.Component; +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; import com.google.common.collect.Lists; /** @@ -76,14 +80,14 @@ import com.google.common.collect.Lists; * @author yangli9 */ @Component("cubeMgmtService") -public class CubeService extends BasicService { +public class CubeService extends BasicService implements InitializingBean { private static final Logger logger = LoggerFactory.getLogger(CubeService.class); public static final char[] VALID_CUBENAME = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_" .toCharArray(); -private WeakHashMap htableInfoCache = new WeakHashMap<>(); +protected Cache htableInfoCache = CacheBuilder.newBuilder().build(); @Autowired @Qualifier("accessService") @@ -101,7 +105,7 @@ public class CubeService extends BasicService { private AclEvaluate aclEvaluate; public List listAllCubes(final String cubeName, final String projectName, final String modelName, - boolean exactMatch) { +boolean exactMatch) { List cubeInstances = null; ProjectInstance project = (null != projectName) ? getProjectManager().getProject(projectName) : null; @@ -414,11 +418,12 @@ public class CubeService extends BasicService { * @throws IOException Exception when HTable resource is not closed correctly. */ public HBaseResponse getHTableInfo(String tableName) throws IOException { -if (htableInfoCache.containsKey(tableName)) { -return htableInfoCache.get(tableName); +HBaseResponse hr = htableInfoCache.getIfPresent(tableName); +if (null != hr) { +return hr; } -
[21/50] [abbrv] kylin git commit: minor, add logs
minor, add logs Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a637d2a0 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a637d2a0 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a637d2a0 Branch: refs/heads/master Commit: a637d2a0cea88d85a372bc449269a50a1502471f Parents: 68ae79a Author: Cheng WangAuthored: Tue Aug 22 16:18:17 2017 +0800 Committer: æ Committed: Tue Aug 22 16:46:53 2017 +0800 -- .../main/java/org/apache/kylin/rest/service/CubeService.java| 1 + .../kylin/storage/hbase/util/HBaseRegionSizeCalculator.java | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/a637d2a0/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java -- diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java index f8ca38d..fe8fb6c 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java +++ b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java @@ -718,6 +718,7 @@ public class CubeService extends BasicService implements InitializingBean { @Override public void afterPropertiesSet() throws Exception { Broadcaster.getInstance(getConfig()).registerListener(new HTableInfoSyncListener(), "cube"); +logger.info("HTableInfoSyncListener is on."); } private class HTableInfoSyncListener extends Broadcaster.Listener { http://git-wip-us.apache.org/repos/asf/kylin/blob/a637d2a0/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseRegionSizeCalculator.java -- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseRegionSizeCalculator.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseRegionSizeCalculator.java index 1daca0a..5189628 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseRegionSizeCalculator.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseRegionSizeCalculator.java @@ -104,8 +104,9 @@ public class HBaseRegionSizeCalculator { sizeMap.put(regionId, regionSizeBytes); countMap.put(regionId, new Pair<>(regionLoad.getStores(), regionLoad.getStorefiles())); -// logger.info("Region " + regionLoad.getNameAsString() -// + " has size " + regionSizeBytes); +if (regionSizeBytes == 0L) { +logger.info("Region " + regionLoad.getNameAsString() + " has size " + regionSizeBytes); +} } } }
[07/50] [abbrv] kylin git commit: KYLIN-2760 Enhance ACL Logic
KYLIN-2760 Enhance ACL Logic Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7a9f74c8 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7a9f74c8 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7a9f74c8 Branch: refs/heads/master Commit: 7a9f74c8a0f70480f40c6e88e0aeb71824fd7884 Parents: fa7eb69 Author: tttMelody <245915...@qq.com> Authored: Wed Aug 16 14:35:08 2017 +0800 Committer: Hongbin MaCommitted: Wed Aug 16 16:04:09 2017 +0800 -- .../org/apache/kylin/cube/CubeInstance.java | 6 + .../java/org/apache/kylin/job/JobInstance.java | 2 +- .../kylin/metadata/model/DataModelDesc.java | 7 +- .../apache/kylin/rest/constant/Constant.java| 11 +- .../rest/controller/ProjectController.java | 25 +--- .../rest/controller/StreamingController.java| 37 +++--- .../rest/controller2/CubeDescControllerV2.java | 10 +- .../rest/controller2/ModelControllerV2.java | 23 ++-- .../rest/controller2/ModelDescControllerV2.java | 18 +-- .../rest/controller2/StreamingControllerV2.java | 33 ++--- .../rest/controller2/TableControllerV2.java | 2 +- .../kylin/rest/service/AccessService.java | 5 + .../apache/kylin/rest/service/CubeService.java | 83 .../kylin/rest/service/DiagnosisService.java| 18 ++- .../kylin/rest/service/HybridService.java | 16 ++- .../apache/kylin/rest/service/JobService.java | 35 ++--- .../kylin/rest/service/KafkaConfigService.java | 25 ++-- .../apache/kylin/rest/service/ModelService.java | 71 ++ .../kylin/rest/service/ProjectService.java | 29 +--- .../apache/kylin/rest/service/QueryService.java | 10 +- .../kylin/rest/service/StreamingService.java| 22 ++-- .../apache/kylin/rest/service/TableService.java | 27 ++-- .../org/apache/kylin/rest/util/AclEvaluate.java | 131 +++ .../org/apache/kylin/rest/util/AclUtil.java | 39 -- .../org/apache/kylin/rest/bean/BeanTest.java| 7 +- .../rest/controller/AccessControllerTest.java | 25 +++- 26 files changed, 440 insertions(+), 277 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/7a9f74c8/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java -- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java index 4d205eb..4c57db8 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java @@ -36,6 +36,8 @@ import org.apache.kylin.metadata.model.MeasureDesc; import org.apache.kylin.metadata.model.SegmentStatusEnum; import org.apache.kylin.metadata.model.Segments; import org.apache.kylin.metadata.model.TblColRef; +import org.apache.kylin.metadata.project.ProjectInstance; +import org.apache.kylin.metadata.project.ProjectManager; import org.apache.kylin.metadata.realization.CapabilityResult; import org.apache.kylin.metadata.realization.CapabilityResult.CapabilityInfluence; import org.apache.kylin.metadata.realization.IRealization; @@ -385,6 +387,10 @@ public class CubeInstance extends RootPersistentEntity implements IRealization, return getDescriptor().getProject(); } +public ProjectInstance getProjectInstance() { +return ProjectManager.getInstance(getConfig()).getProject(getProject()); +} + @Override public int getSourceType() { return getModel().getRootFactTable().getTableDesc().getSourceType(); http://git-wip-us.apache.org/repos/asf/kylin/blob/7a9f74c8/core-job/src/main/java/org/apache/kylin/job/JobInstance.java -- diff --git a/core-job/src/main/java/org/apache/kylin/job/JobInstance.java b/core-job/src/main/java/org/apache/kylin/job/JobInstance.java index 3778834..b94 100644 --- a/core-job/src/main/java/org/apache/kylin/job/JobInstance.java +++ b/core-job/src/main/java/org/apache/kylin/job/JobInstance.java @@ -167,7 +167,7 @@ public class JobInstance extends RootPersistentEntity implements Comparablehttp://git-wip-us.apache.org/repos/asf/kylin/blob/7a9f74c8/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java -- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java index f82c996..2037420 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java @@ -40,6 +40,7
[28/50] [abbrv] kylin git commit: KYLIN-2809 update calcite to support "+" as string concat
KYLIN-2809 update calcite to support "+" as string concat Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d059c05e Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d059c05e Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d059c05e Branch: refs/heads/master Commit: d059c05e7a064c6c6bc861fb3e399985ed968177 Parents: d873ca1 Author: Roger ShiAuthored: Thu Aug 24 15:28:10 2017 +0800 Committer: Roger Shi Committed: Thu Aug 24 15:46:12 2017 +0800 -- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/d059c05e/pom.xml -- diff --git a/pom.xml b/pom.xml index 5b00798..d82a7f4 100644 --- a/pom.xml +++ b/pom.xml @@ -67,7 +67,7 @@ -1.13.0-kylin-r1-SNAPSHOT +1.13.0-kylin-r2-SNAPSHOT 1.10.0
[19/50] [abbrv] kylin git commit: KYLIN-2801 Make default precision and scale in DataType (for hive) configurable
KYLIN-2801 Make default precision and scale in DataType (for hive) configurable Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a3d42f11 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a3d42f11 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a3d42f11 Branch: refs/heads/master Commit: a3d42f11924c734b40a7b61fbf6f2f8bac3caa88 Parents: 46cd881 Author: Hongbin MaAuthored: Tue Aug 22 14:12:27 2017 +0800 Committer: Hongbin Ma Committed: Tue Aug 22 16:00:14 2017 +0800 -- .../apache/kylin/common/KylinConfigBase.java| 41 .../kylin/metadata/datatype/DataType.java | 24 ++-- 2 files changed, 54 insertions(+), 11 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/a3d42f11/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java -- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index 0f5f3a9..b622825 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -598,6 +598,47 @@ abstract public class KylinConfigBase implements Serializable { return getOptional("kylin.source.hive.flat-table-cluster-by-dict-column"); } +public int getDefaultVarcharPrecision() { +int v = Integer.parseInt(getOptional("kylin.source.hive.default-varchar-precision", "256")); +if (v < 1) { +return 256; +} else if (v > 65355) { +return 65535; +} else { +return v; +} +} + +public int getDefaultCharPrecision() { +//at most 255 according to https://cwiki.apache.org/confluence/display/Hive/LanguageManual+Types#LanguageManualTypes-CharcharChar +int v = Integer.parseInt(getOptional("kylin.source.hive.default-char-precision", "255")); +if (v < 1) { +return 255; +} else if (v > 255) { +return 255; +} else { +return v; +} +} + +public int getDefaultDecimalPrecision() { +int v = Integer.parseInt(getOptional("kylin.source.hive.default-decimal-precision", "19")); +if (v < 1) { +return 19; +} else { +return v; +} +} + +public int getDefaultDecimalScale() { +int v = Integer.parseInt(getOptional("kylin.source.hive.default-decimal-scale", "4")); +if (v < 1) { +return 4; +} else { +return v; +} +} + // // SOURCE.KAFKA // http://git-wip-us.apache.org/repos/asf/kylin/blob/a3d42f11/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataType.java -- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataType.java b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataType.java index 66ff581..13cc2be 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataType.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataType.java @@ -31,6 +31,7 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; import org.apache.commons.lang.StringUtils; +import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.util.BytesSerializer; import org.apache.kylin.common.util.BytesUtil; import org.apache.kylin.measure.MeasureTypeFactory; @@ -177,21 +178,22 @@ public class DataType implements Serializable { } } -// FIXME 256 for unknown string precision -if ((name.equals("char") || name.equals("varchar")) && precision == -1) { -precision = 256; // to save memory at frontend, e.g. tableau will - // allocate memory according to this +if (precision == -1) { +// FIXME 256 for unknown string precision + +// why 256(255) as default? +// to save memory at frontend, e.g. tableau will +// allocate memory according to this if (name.equals("char")) { -precision -= 1; //at most 255 according to https://cwiki.apache.org/confluence/display/Hive/LanguageManual+Types#LanguageManualTypes-CharcharChar +precision = KylinConfig.getInstanceFromEnv().getDefaultCharPrecision(); +} else if (name.equals("varchar"))
[16/50] [abbrv] kylin git commit: KYLIN-2575 code refactors
http://git-wip-us.apache.org/repos/asf/kylin/blob/aed840f9/query/src/main/java/org/apache/kylin/query/relnode/ColumnRowType.java -- diff --git a/query/src/main/java/org/apache/kylin/query/relnode/ColumnRowType.java b/query/src/main/java/org/apache/kylin/query/relnode/ColumnRowType.java index f2894e7..ce50122 100644 --- a/query/src/main/java/org/apache/kylin/query/relnode/ColumnRowType.java +++ b/query/src/main/java/org/apache/kylin/query/relnode/ColumnRowType.java @@ -56,6 +56,10 @@ public class ColumnRowType { return columns.get(index); } +public TblColRef getColumnByName(String columnName) { +return getColumnByIndexNullable(getIndexByName(columnName)); +} + public int getIndexByName(String columnName) { for (int i = 0; i < columns.size(); i++) { if (columns.get(i).getName().equals(columnName)) { http://git-wip-us.apache.org/repos/asf/kylin/blob/aed840f9/query/src/main/java/org/apache/kylin/query/relnode/OLAPProjectRel.java -- diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPProjectRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPProjectRel.java index e7b09a3..8e93659 100644 --- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPProjectRel.java +++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPProjectRel.java @@ -158,7 +158,7 @@ public class OLAPProjectRel extends Project implements OLAPRel { RexCall call = (RexCall) rexNode; column = translateRexCall(call, inputColumnRowType, fieldName, sourceCollector); } else { -throw new IllegalStateException("Unsupport RexNode " + rexNode); +throw new IllegalStateException("Unsupported RexNode " + rexNode); } return column; } http://git-wip-us.apache.org/repos/asf/kylin/blob/aed840f9/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java -- diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java index b583291..7801891 100644 --- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java +++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java @@ -238,7 +238,7 @@ public class OLAPTableScan extends TableScan implements OLAPRel, EnumerableRel { TableRef tableRef = TblColRef.tableForUnknownModel(this.alias, olapTable.getSourceTable()); Listcolumns = new ArrayList (); -for (ColumnDesc sourceColumn : olapTable.getExposedColumns()) { +for (ColumnDesc sourceColumn : olapTable.getSourceColumns()) { TblColRef colRef = TblColRef.columnForUnknownModel(tableRef, sourceColumn); columns.add(colRef); } http://git-wip-us.apache.org/repos/asf/kylin/blob/aed840f9/query/src/main/java/org/apache/kylin/query/schema/OLAPSchema.java -- diff --git a/query/src/main/java/org/apache/kylin/query/schema/OLAPSchema.java b/query/src/main/java/org/apache/kylin/query/schema/OLAPSchema.java index fe897e9..5387a74 100644 --- a/query/src/main/java/org/apache/kylin/query/schema/OLAPSchema.java +++ b/query/src/main/java/org/apache/kylin/query/schema/OLAPSchema.java @@ -41,6 +41,7 @@ public class OLAPSchema extends AbstractSchema { private KylinConfig config; private String projectName; private String schemaName; +private boolean exposeMore; private StorageURL storageUrl; private String starSchemaUrl; private String starSchemaUser; @@ -54,9 +55,10 @@ public class OLAPSchema extends AbstractSchema { this.starSchemaPassword = config.getHivePassword(); } -public OLAPSchema(String project, String schemaName) { +public OLAPSchema(String project, String schemaName, boolean exposeMore) { this.projectName = ProjectInstance.getNormalizedProjectName(project); this.schemaName = schemaName; +this.exposeMore = exposeMore; init(); } @@ -66,19 +68,21 @@ public class OLAPSchema extends
[12/50] [abbrv] kylin git commit: minor, fix query export garbled when results having Chinese
minor, fix query export garbled when results having Chinese Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8906d133 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8906d133 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8906d133 Branch: refs/heads/master Commit: 8906d133f430254ed4a8c1f6f7b989b0c4ca2c09 Parents: 1c8de93 Author: Jiatao Tao <245915...@qq.com> Authored: Mon Aug 21 02:08:08 2017 -0500 Committer: GitHubCommitted: Mon Aug 21 02:08:08 2017 -0500 -- .../apache/kylin/rest/controller2/QueryControllerV2.java | 10 -- 1 file changed, 8 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/8906d133/server-base/src/main/java/org/apache/kylin/rest/controller2/QueryControllerV2.java -- diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller2/QueryControllerV2.java b/server-base/src/main/java/org/apache/kylin/rest/controller2/QueryControllerV2.java index 5809d6e..1aa72ba 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/controller2/QueryControllerV2.java +++ b/server-base/src/main/java/org/apache/kylin/rest/controller2/QueryControllerV2.java @@ -19,6 +19,9 @@ package org.apache.kylin.rest.controller2; import java.io.IOException; +import java.io.OutputStreamWriter; +import java.io.Writer; +import java.nio.charset.StandardCharsets; import java.sql.SQLException; import java.text.SimpleDateFormat; import java.util.ArrayList; @@ -159,7 +162,7 @@ public class QueryControllerV2 extends BasicController { "application/vnd.apache.kylin-v2+json" }, consumes = MediaType.APPLICATION_FORM_URLENCODED_VALUE) @ResponseBody public void downloadQueryResultV2(@PathVariable String format, SQLRequest sqlRequest, -HttpServletResponse response) { +HttpServletResponse response) throws IOException { SQLResponse result = queryService.doQueryWithCache(sqlRequest); @@ -170,8 +173,11 @@ public class QueryControllerV2 extends BasicController { ICsvListWriter csvWriter = null; try { -csvWriter = new CsvListWriter(response.getWriter(), CsvPreference.STANDARD_PREFERENCE); +//Add a BOM for Excel +Writer writer = new OutputStreamWriter(response.getOutputStream(), StandardCharsets.UTF_8); +writer.write('\uFEFF'); +csvWriter = new CsvListWriter(writer, CsvPreference.STANDARD_PREFERENCE); List headerList = new ArrayList(); for (SelectedColumnMeta column : result.getColumnMetas()) {
[23/50] [abbrv] kylin git commit: minor, only apply adjustSqlDigest to involved aggregations
minor, only apply adjustSqlDigest to involved aggregations Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d85f6ab0 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d85f6ab0 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d85f6ab0 Branch: refs/heads/master Commit: d85f6ab0130e3c1962c9dc770e06f178d3d85c32 Parents: f26b1ab Author: Roger ShiAuthored: Tue Aug 22 17:42:37 2017 +0800 Committer: Roger Shi Committed: Tue Aug 22 17:55:53 2017 +0800 -- .../org/apache/kylin/cube/CubeCapabilityChecker.java | 5 + .../extendedcolumn/ExtendedColumnMeasureType.java| 10 +- .../org/apache/kylin/measure/raw/RawMeasureType.java | 10 +- .../apache/kylin/measure/topn/TopNMeasureType.java | 15 ++- .../kylin/metadata/realization/CapabilityResult.java | 8 .../apache/kylin/metadata/realization/SQLDigest.java | 10 -- .../apache/kylin/storage/hbase/ITStorageTest.java| 4 +++- .../kylin/query/enumerator/OLAPEnumerator.java | 1 - .../org/apache/kylin/query/relnode/OLAPContext.java | 5 - .../org/apache/kylin/query/routing/QueryRouter.java | 6 ++ 10 files changed, 66 insertions(+), 8 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/d85f6ab0/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java -- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java index 61516d2..47f268a 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java @@ -127,6 +127,11 @@ public class CubeCapabilityChecker { public double suggestCostMultiplier() { return 100; } + +@Override +public MeasureDesc getInvolvedMeasure() { +return null; +} }); } http://git-wip-us.apache.org/repos/asf/kylin/blob/d85f6ab0/core-metadata/src/main/java/org/apache/kylin/measure/extendedcolumn/ExtendedColumnMeasureType.java -- diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/extendedcolumn/ExtendedColumnMeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/extendedcolumn/ExtendedColumnMeasureType.java index de5ee25..c95cd1c 100644 --- a/core-metadata/src/main/java/org/apache/kylin/measure/extendedcolumn/ExtendedColumnMeasureType.java +++ b/core-metadata/src/main/java/org/apache/kylin/measure/extendedcolumn/ExtendedColumnMeasureType.java @@ -95,6 +95,9 @@ public class ExtendedColumnMeasureType extends MeasureType { @Override public void adjustSqlDigest(List measureDescs, SQLDigest sqlDigest) { for (MeasureDesc measureDesc : measureDescs) { +if (!sqlDigest.involvedMeasure.contains(measureDesc)) { +continue; +} FunctionDesc extendColumnFunc = measureDesc.getFunction(); List hosts = getExtendedColumnHosts(extendColumnFunc); TblColRef extended = getExtendedColumn(extendColumnFunc); @@ -111,7 +114,7 @@ public class ExtendedColumnMeasureType extends MeasureType { } @Override -public CapabilityResult.CapabilityInfluence influenceCapabilityCheck(Collection unmatchedDimensions, Collection unmatchedAggregations, SQLDigest digest, MeasureDesc measureDesc) { +public CapabilityResult.CapabilityInfluence influenceCapabilityCheck(Collection unmatchedDimensions, Collection unmatchedAggregations, SQLDigest digest, final MeasureDesc measureDesc) { TblColRef extendedCol = getExtendedColumn(measureDesc.getFunction()); if (!unmatchedDimensions.contains(extendedCol)) { @@ -129,6 +132,11 @@ public class ExtendedColumnMeasureType extends MeasureType { public double suggestCostMultiplier() { return 0.9; } + +@Override +public MeasureDesc getInvolvedMeasure() { +return measureDesc; +} }; } http://git-wip-us.apache.org/repos/asf/kylin/blob/d85f6ab0/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java -- diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java index 31f35e9..eee8fa6 100644 ---
[04/50] [abbrv] kylin git commit: #1949, fix NPE when discarding model check job (#1952)
#1949, fix NPE when discarding model check job (#1952) Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c487fe49 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c487fe49 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c487fe49 Branch: refs/heads/master Commit: c487fe4990473afa11d30a2e16ceb133e3f25b6d Parents: 2eca43c Author: æAuthored: Wed Aug 16 12:05:20 2017 +0800 Committer: luguosheng1314 <550175...@qq.com> Committed: Wed Aug 16 12:05:20 2017 +0800 -- .../apache/kylin/rest/service/JobService.java | 127 +-- 1 file changed, 63 insertions(+), 64 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/c487fe49/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java -- diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java b/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java index 6451d66..5dafa08 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java +++ b/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java @@ -18,18 +18,12 @@ package org.apache.kylin.rest.service; -import java.io.IOException; -import java.util.Calendar; -import java.util.Collections; -import java.util.Date; -import java.util.EnumSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.TimeZone; - -import javax.annotation.Nullable; - +import com.google.common.base.Function; +import com.google.common.base.Predicate; +import com.google.common.base.Predicates; +import com.google.common.collect.FluentIterable; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; import org.apache.commons.lang3.StringUtils; import org.apache.directory.api.util.Strings; import org.apache.kylin.common.KylinConfig; @@ -72,12 +66,16 @@ import org.springframework.context.annotation.EnableAspectJAutoProxy; import org.springframework.security.access.prepost.PreAuthorize; import org.springframework.stereotype.Component; -import com.google.common.base.Function; -import com.google.common.base.Predicate; -import com.google.common.base.Predicates; -import com.google.common.collect.FluentIterable; -import com.google.common.collect.Lists; -import com.google.common.collect.Sets; +import javax.annotation.Nullable; +import java.io.IOException; +import java.util.Calendar; +import java.util.Collections; +import java.util.Date; +import java.util.EnumSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TimeZone; /** * @author ysong1 @@ -158,22 +156,22 @@ public class JobService extends BasicService implements InitializingBean { Message msg = MsgPicker.getMsg(); switch (status) { -case DISCARDED: -return ExecutableState.DISCARDED; -case ERROR: -return ExecutableState.ERROR; -case FINISHED: -return ExecutableState.SUCCEED; -case NEW: -return ExecutableState.READY; -case PENDING: -return ExecutableState.READY; -case RUNNING: -return ExecutableState.RUNNING; -case STOPPED: -return ExecutableState.STOPPED; -default: -throw new BadRequestException(String.format(msg.getILLEGAL_EXECUTABLE_STATE(), status)); +case DISCARDED: +return ExecutableState.DISCARDED; +case ERROR: +return ExecutableState.ERROR; +case FINISHED: +return ExecutableState.SUCCEED; +case NEW: +return ExecutableState.READY; +case PENDING: +return ExecutableState.READY; +case RUNNING: +return ExecutableState.RUNNING; +case STOPPED: +return ExecutableState.STOPPED; +default: +throw new BadRequestException(String.format(msg.getILLEGAL_EXECUTABLE_STATE(), status)); } } @@ -181,30 +179,30 @@ public class JobService extends BasicService implements InitializingBean { Message msg = MsgPicker.getMsg(); switch (timeFilter) { -case LAST_ONE_DAY: -calendar.add(Calendar.DAY_OF_MONTH, -1); -return calendar.getTimeInMillis(); -case LAST_ONE_WEEK: -calendar.add(Calendar.WEEK_OF_MONTH, -1); -return calendar.getTimeInMillis(); -case LAST_ONE_MONTH: -calendar.add(Calendar.MONTH, -1); -return calendar.getTimeInMillis(); -case LAST_ONE_YEAR: -calendar.add(Calendar.YEAR, -1); -return
[02/50] [abbrv] kylin git commit: KYLIN-2783 merge AggregationGroupScheduler into CuboidScheduler
KYLIN-2783 merge AggregationGroupScheduler into CuboidScheduler Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c8bab3df Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c8bab3df Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c8bab3df Branch: refs/heads/master Commit: c8bab3df33edd5dc146a02f94caccd0d810b68bd Parents: 5008866 Author: Li YangAuthored: Sun Aug 13 16:34:22 2017 +0800 Committer: Roger Shi Committed: Mon Aug 14 09:35:09 2017 +0800 -- .../org/apache/kylin/common/KylinVersion.java | 24 ++- .../apache/kylin/common/KylinVersionTest.java | 19 +- .../cube/cuboid/AggregationGroupScheduler.java | 126 .../kylin/cube/cuboid/CuboidScheduler.java | 4 + .../cube/cuboid/DefaultCuboidScheduler.java | 191 ++- .../kylin/cube/model/AggregationGroup.java | 10 +- .../org/apache/kylin/cube/model/SelectRule.java | 6 +- 7 files changed, 187 insertions(+), 193 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/c8bab3df/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java -- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java b/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java index 960bafe..c6913fc 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java @@ -37,10 +37,11 @@ public class KylinVersion implements Comparable { private static final String COMMIT_SHA1_v15 = "commit_SHA1"; private static final String COMMIT_SHA1_v13 = "commit.sha1"; -public int major; -public int minor; -public int revision; -public boolean isSnapshot; +public final int major; +public final int minor; +public final int revision; +public final int internal; +public final boolean isSnapshot; public KylinVersion(String version) { @@ -58,12 +59,13 @@ public class KylinVersion implements Comparable { major = Integer.parseInt(splits[0]); minor = Integer.parseInt(splits[1]); -revision = splits.length < 3 ? 0 : Integer.parseInt(splits[2]); +revision = splits.length > 2 ? Integer.parseInt(splits[2]) : 0; +internal = splits.length > 3 ? Integer.parseInt(splits[3]) : 0; } @Override public String toString() { -return "" + major + "." + minor + "." + revision; +return "" + major + "." + minor + "." + revision + "." + internal + (isSnapshot ? "-SNAPSHOT" : ""); } @Override @@ -83,13 +85,17 @@ public class KylinVersion implements Comparable { if (comp != 0) return comp; +comp = this.internal - v.internal; +if (comp != 0) +return comp; + return (this.isSnapshot ? 0 : 1) - (v.isSnapshot ? 0 : 1); } /** * Require MANUAL updating kylin version per ANY upgrading. */ -private static final KylinVersion CURRENT_KYLIN_VERSION = new KylinVersion("2.1.0"); +private static final KylinVersion CURRENT_KYLIN_VERSION = new KylinVersion("2.1.0.20403"); private static final KylinVersion VERSION_200 = new KylinVersion("2.0.0"); @@ -125,6 +131,10 @@ public class KylinVersion implements Comparable { public static boolean isBefore200(String ver) { return new KylinVersion(ver).compareTo(VERSION_200) < 0; } + +public static int compare(String v1, String v2) { +return new KylinVersion(v1).compareTo(new KylinVersion(v2)); +} public static void main(String[] args) { System.out.println(getKylinClientInformation()); http://git-wip-us.apache.org/repos/asf/kylin/blob/c8bab3df/core-common/src/test/java/org/apache/kylin/common/KylinVersionTest.java -- diff --git a/core-common/src/test/java/org/apache/kylin/common/KylinVersionTest.java b/core-common/src/test/java/org/apache/kylin/common/KylinVersionTest.java index 96b3189..6e903c8 100644 --- a/core-common/src/test/java/org/apache/kylin/common/KylinVersionTest.java +++ b/core-common/src/test/java/org/apache/kylin/common/KylinVersionTest.java @@ -37,4 +37,21 @@ public class KylinVersionTest { Assert.assertEquals(1, ver1.minor); Assert.assertEquals(0, ver1.revision); } -} + +@Test +public void testToString() { +KylinVersion ver1 = new KylinVersion("2.1.7.321"); +Assert.assertEquals(2, ver1.major); +Assert.assertEquals(1, ver1.minor); +Assert.assertEquals(7, ver1.revision); +Assert.assertEquals(321, ver1.internal); +
[06/50] [abbrv] kylin git commit: KYLIN-2760 Enhance ACL Logic
http://git-wip-us.apache.org/repos/asf/kylin/blob/7a9f74c8/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java -- diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java b/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java index 751d014..30ee99c 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java +++ b/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java @@ -43,11 +43,11 @@ import org.apache.kylin.metadata.model.ColumnDesc; import org.apache.kylin.metadata.model.TableDesc; import org.apache.kylin.metadata.model.TableExtDesc; import org.apache.kylin.metadata.streaming.StreamingConfig; -import org.apache.kylin.rest.constant.Constant; import org.apache.kylin.rest.exception.BadRequestException; import org.apache.kylin.rest.msg.Message; import org.apache.kylin.rest.msg.MsgPicker; import org.apache.kylin.rest.response.TableDescResponse; +import org.apache.kylin.rest.util.AclEvaluate; import org.apache.kylin.source.ISourceMetadataExplorer; import org.apache.kylin.source.SourceFactory; import org.apache.kylin.source.hive.cardinality.HiveColumnCardinalityJob; @@ -57,7 +57,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.security.access.prepost.PreAuthorize; import org.springframework.security.core.context.SecurityContextHolder; import org.springframework.stereotype.Component; @@ -88,27 +87,34 @@ public class TableService extends BasicService { @Qualifier("kafkaMgmtService") private KafkaConfigService kafkaConfigService; +@Autowired +private AclEvaluate aclEvaluate; + public List getTableDescByProject(String project, boolean withExt) throws IOException { +aclEvaluate.checkProjectReadPermission(project); List tables = getProjectManager().listDefinedTables(project); if (null == tables) { return Collections.emptyList(); } if (withExt) { +aclEvaluate.checkProjectWritePermission(project); tables = cloneTableDesc(tables, project); } return tables; } public TableDesc getTableDescByName(String tableName, boolean withExt, String prj) { +aclEvaluate.checkProjectReadPermission(prj); TableDesc table = getMetadataManager().getTableDesc(tableName, prj); if (withExt) { +aclEvaluate.checkProjectWritePermission(prj); table = cloneTableDesc(table, prj); } return table; } -@PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN) public String[] loadHiveTablesToProject(String[] tables, String project) throws Exception { +aclEvaluate.checkProjectWritePermission(project); // de-dup SetMultimapdb2tables = LinkedHashMultimap.create(); for (String fullTableName : tables) { @@ -176,6 +182,7 @@ public class TableService extends BasicService { } public Map loadHiveTables(String[] tableNames, String project, boolean isNeedProfile) throws Exception { +aclEvaluate.checkProjectWritePermission(project); String submitter = SecurityContextHolder.getContext().getAuthentication().getName(); Map result = new HashMap (); @@ -198,6 +205,7 @@ public class TableService extends BasicService { } public Map unloadHiveTables(String[] tableNames, String project) throws IOException { +aclEvaluate.checkProjectWritePermission(project); Set unLoadSuccess = Sets.newHashSet(); Set unLoadFail = Sets.newHashSet(); Map result = new HashMap (); @@ -231,8 +239,8 @@ public class TableService extends BasicService { * @param project * @return */ -@PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN) public boolean unLoadHiveTable(String tableName, String project) throws IOException { +aclEvaluate.checkProjectWritePermission(project); Message msg = MsgPicker.getMsg(); boolean rtn = false; @@ -266,9 +274,9 @@ public class TableService extends BasicService { KafkaConfig kafkaConfig = null; try { config = streamingService.getStreamingManager().getStreamingConfig(tableName); -kafkaConfig = kafkaConfigService.getKafkaConfig(tableName); -streamingService.dropStreamingConfig(config); -kafkaConfigService.dropKafkaConfig(kafkaConfig); +kafkaConfig = kafkaConfigService.getKafkaConfig(tableName, project); +streamingService.dropStreamingConfig(config, project); +
[13/50] [abbrv] kylin git commit: minor, unknown precision for char should be 255
minor, unknown precision for char should be 255 Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/909628ff Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/909628ff Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/909628ff Branch: refs/heads/master Commit: 909628ff6d128639eb813d616fe7281295378619 Parents: 788c731 Author: Hongbin MaAuthored: Mon Aug 21 22:21:25 2017 +0800 Committer: Hongbin Ma Committed: Mon Aug 21 22:55:59 2017 +0800 -- .../org/apache/kylin/metadata/datatype/DataType.java | 11 --- 1 file changed, 8 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/909628ff/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataType.java -- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataType.java b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataType.java index 83b2391..66ff581 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataType.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataType.java @@ -164,14 +164,16 @@ public class DataType implements Serializable { try { n = Integer.parseInt(parts[i]); } catch (NumberFormatException e) { -throw new IllegalArgumentException("bad data type -- " + datatype + ", precision/scale not numeric"); +throw new IllegalArgumentException( +"bad data type -- " + datatype + ", precision/scale not numeric"); } if (i == 0) precision = n; else if (i == 1) scale = n; else -throw new IllegalArgumentException("bad data type -- " + datatype + ", too many precision/scale parts"); +throw new IllegalArgumentException( +"bad data type -- " + datatype + ", too many precision/scale parts"); } } @@ -179,6 +181,9 @@ public class DataType implements Serializable { if ((name.equals("char") || name.equals("varchar")) && precision == -1) { precision = 256; // to save memory at frontend, e.g. tableau will // allocate memory according to this +if (name.equals("char")) { +precision -= 1; //at most 255 according to https://cwiki.apache.org/confluence/display/Hive/LanguageManual+Types#LanguageManualTypes-CharcharChar +} } // FIXME (19,4) for unknown decimal precision @@ -217,7 +222,7 @@ public class DataType implements Serializable { public boolean isTimeFamily() { return DATETIME_FAMILY.contains(name) && !isDate(); } - + public boolean isDate() { return name.equals("date"); }
[25/50] [abbrv] kylin git commit: KYLIN-2776 Using dropwizard as default metric framework
KYLIN-2776 Using dropwizard as default metric framework Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8e078847 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8e078847 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8e078847 Branch: refs/heads/master Commit: 8e078847a42b4f7853a2c1a75afa405c09c4ea7b Parents: d85f6ab Author: yiming.xu <100650...@qq.com> Authored: Wed Aug 23 13:12:58 2017 +0800 Committer: Hongbin MaCommitted: Wed Aug 23 13:49:23 2017 +0800 -- core-common/pom.xml | 17 + .../apache/kylin/common/KylinConfigBase.java| 20 + .../kylin/common/metrics/common/Metrics.java| 162 ++ .../common/metrics/common/MetricsConstant.java | 38 ++ .../common/metrics/common/MetricsFactory.java | 62 +++ .../common/metrics/common/MetricsScope.java | 33 ++ .../common/metrics/common/MetricsVariable.java | 27 + .../kylin/common/metrics/common/Metricss.java | 40 ++ .../metrics/metrics2/CodahaleMetrics.java | 508 +++ .../metrics/metrics2/CodahaleReporter.java | 31 ++ .../metrics2/ConsoleMetricsReporter.java| 51 ++ .../metrics/metrics2/JmxMetricsReporter.java| 55 ++ .../metrics2/JsonFileMetricsReporter.java | 136 + .../metrics2/KylinObjectNameFactory.java| 68 +++ .../metrics2/MetricVariableRatioGauge.java | 47 ++ .../metrics/metrics2/Metrics2Reporter.java | 60 +++ .../metrics/metrics2/MetricsReporting.java | 26 + .../common/metrics/perflog/IPerfLogger.java | 48 ++ .../common/metrics/perflog/PerfLogger.java | 158 ++ .../metrics/perflog/PerfLoggerFactory.java | 56 ++ .../metrics/perflog/SimplePerfLogger.java | 73 +++ pom.xml | 3 +- .../kylin/rest/init/InitialTaskManager.java | 4 +- .../kylin/rest/metrics/QueryMetrics2Facade.java | 106 .../apache/kylin/rest/service/QueryService.java | 2 + .../kylin/rest/metrics/QueryMetrics2Test.java | 128 + 26 files changed, 1957 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/8e078847/core-common/pom.xml -- diff --git a/core-common/pom.xml b/core-common/pom.xml index 8852743..d58de36 100644 --- a/core-common/pom.xml +++ b/core-common/pom.xml @@ -68,6 +68,23 @@ junit test + + +io.dropwizard.metrics +metrics-jvm +${dropwizard.version} + + +io.dropwizard.metrics +metrics-json +${dropwizard.version} + + + +com.github.joshelser + dropwizard-metrics-hadoop-metrics2-reporter +0.1.2 + http://git-wip-us.apache.org/repos/asf/kylin/blob/8e078847/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java -- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index b622825..99b9bb2 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -1173,4 +1173,24 @@ abstract public class KylinConfigBase implements Serializable { public boolean isWebCrossDomainEnabled() { return Boolean.parseBoolean(getOptional("kylin.web.cross-domain-enabled", "true")); } + +/** + * metric + */ +public String getCoadhaleMetricReportClassesName() { +return getOptional("kylin.metric.codahale-metric-report-classes", + "org.apache.kylin.common.metrics.metrics2.JsonFileMetricsReporter,org.apache.kylin.common.metrics.metrics2.JmxMetricsReporter"); +} + +public String getMetricFileLocation() { +return getOptional("kylin.metric.file.location", "/tmp/report.json"); +} + +public Long getJsonFileMetricsReporterInterval() { +return Long.parseLong(getOptional("kylin.metric.json-file-metric-reporter.interval", "5000")); +} + +public String getPerfLoggerClassName() { +return getOptional("kylin.metric.perf-logger.class", "org.apache.kylin.common.metrics.perflog.PerfLogger"); +} } http://git-wip-us.apache.org/repos/asf/kylin/blob/8e078847/core-common/src/main/java/org/apache/kylin/common/metrics/common/Metrics.java -- diff --git a/core-common/src/main/java/org/apache/kylin/common/metrics/common/Metrics.java
[29/50] [abbrv] kylin git commit: minor, fix some files license header
minor, fix some files license header Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1fff4214 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1fff4214 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1fff4214 Branch: refs/heads/master Commit: 1fff42146aa539b599802f9b19de28dc87d05fad Parents: d059c05 Author: Roger ShiAuthored: Fri Aug 25 17:31:19 2017 +0800 Committer: Roger Shi Committed: Fri Aug 25 17:36:24 2017 +0800 -- .../calcite/prepare/CalcitePrepareImpl.java | 34 .../calcite/sql2rel/SqlToRelConverter.java | 34 .../java/org/apache/calcite/tools/Programs.java | 34 3 files changed, 42 insertions(+), 60 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/1fff4214/atopcalcite/src/main/java/org/apache/calcite/prepare/CalcitePrepareImpl.java -- diff --git a/atopcalcite/src/main/java/org/apache/calcite/prepare/CalcitePrepareImpl.java b/atopcalcite/src/main/java/org/apache/calcite/prepare/CalcitePrepareImpl.java index b1ce7ea..7b4c549 100644 --- a/atopcalcite/src/main/java/org/apache/calcite/prepare/CalcitePrepareImpl.java +++ b/atopcalcite/src/main/java/org/apache/calcite/prepare/CalcitePrepareImpl.java @@ -1,26 +1,20 @@ /* - * Copyright (C) 2016 Kyligence Inc. All rights reserved. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at * - * http://kyligence.io + * http://www.apache.org/licenses/LICENSE-2.0 * - * This software is the confidential and proprietary information of - * Kyligence Inc. ("Confidential Information"). You shall not disclose - * such Confidential Information and shall use it only in accordance - * with the terms of the license agreement you entered into with - * Kyligence Inc. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ package org.apache.calcite.prepare; import static org.apache.calcite.util.Static.RESOURCE; http://git-wip-us.apache.org/repos/asf/kylin/blob/1fff4214/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java -- diff --git a/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java b/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java index 18aadef..f10ba96 100644 --- a/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java +++ b/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java @@ -1,26 +1,20 @@ /* - * Copyright (C) 2016 Kyligence Inc. All rights reserved. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at * - * http://kyligence.io + * http://www.apache.org/licenses/LICENSE-2.0 * - * This software is the confidential and proprietary information of - * Kyligence Inc. ("Confidential Information"). You shall not disclose -
[24/50] [abbrv] kylin git commit: KYLIN-2776 Using dropwizard as default metric framework
http://git-wip-us.apache.org/repos/asf/kylin/blob/8e078847/server/src/test/java/org/apache/kylin/rest/metrics/QueryMetrics2Test.java -- diff --git a/server/src/test/java/org/apache/kylin/rest/metrics/QueryMetrics2Test.java b/server/src/test/java/org/apache/kylin/rest/metrics/QueryMetrics2Test.java new file mode 100644 index 000..5e3e00b --- /dev/null +++ b/server/src/test/java/org/apache/kylin/rest/metrics/QueryMetrics2Test.java @@ -0,0 +1,128 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kylin.rest.metrics; + +import static org.apache.kylin.common.metrics.common.Metricss.buildCubeMetricPrefix; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; + +import javax.management.MBeanServer; +import javax.management.ObjectName; + +import org.apache.kylin.common.metrics.common.Metrics; +import org.apache.kylin.common.metrics.common.MetricsConstant; +import org.apache.kylin.common.metrics.common.MetricsFactory; +import org.apache.kylin.common.metrics.common.Metricss; +import org.apache.kylin.rest.request.SQLRequest; +import org.apache.kylin.rest.response.SQLResponse; +import org.apache.kylin.rest.service.ServiceTestBase; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import com.codahale.metrics.Snapshot; + +public class QueryMetrics2Test extends ServiceTestBase { + +private static MBeanServer mBeanServer; +private static ObjectName objectName; + +@Before +public void setup() throws Exception { +super.setup(); + +objectName = new ObjectName("Hadoop:service=Kylin,name=Server_Total"); +} + +@Test +public void testQueryMetrics() throws Exception { +System.setProperty("kylin.server.query-metrics-enabled", "true"); +QueryMetrics2Facade.init(); + +SQLRequest sqlRequest = new SQLRequest(); +sqlRequest.setSql("select * from TEST_KYLIN_FACT"); +sqlRequest.setProject("default"); + +SQLResponse sqlResponse = new SQLResponse(); +sqlResponse.setDuration(10); +sqlResponse.setCube("test_cube"); +sqlResponse.setIsException(false); +sqlResponse.setTotalScanCount(100); +List list1 = new ArrayList<>(); +list1.add("111"); +list1.add("112"); +List list2 = new ArrayList<>(); +list2.add("111"); +list2.add("112"); +Listresults = new ArrayList<>(); +results.add(list1); +results.add(list2); +sqlResponse.setResults(results); +sqlResponse.setStorageCacheUsed(true); +Metrics metrics = MetricsFactory.getInstance(); +QueryMetrics2Facade.updateMetrics(sqlRequest, sqlResponse); +String prefix = buildCubeMetricPrefix(sqlRequest.getProject(), sqlResponse.getCube().replace("=", "->")); +Thread.sleep(2000); + +Assert.assertEquals(1L, +metrics.getCounter(Metricss.buildMetricName(prefix, MetricsConstant.QUERY_COUNT)).getCount()); +Assert.assertEquals(1L, +metrics.getCounter(Metricss.buildMetricName(prefix, MetricsConstant.QUERY_SUCCESS_COUNT)).getCount()); +Assert.assertEquals(0L, +metrics.getCounter(Metricss.buildMetricName(prefix, MetricsConstant.QUERY_FAIL_COUNT)).getCount()); +Assert.assertEquals(1L, +metrics.getCounter(Metricss.buildMetricName(prefix, MetricsConstant.QUERY_CACHE_COUNT)).getCount()); +// +Snapshot queryScanSnapshot= metrics +.getHistogram(Metricss.buildMetricName(prefix, MetricsConstant.QUERY_SCAN_ROWCOUNT)).getSnapshot(); +Assert.assertEquals(100.0, queryScanSnapshot.getMean(), 0); +Assert.assertEquals(100.0, queryScanSnapshot.getMax(), 0); +Assert.assertEquals(100.0, queryScanSnapshot.getMin(), 0); + +Snapshot queryResultSnapshot = metrics +.getHistogram(Metricss.buildMetricName(prefix, MetricsConstant.QUERY_RESULT_ROWCOUNT)).getSnapshot(); + +Assert.assertEquals(2.0, queryResultSnapshot.getMean(), 0); +Assert.assertEquals(2.0,
svn commit: r1806793 [2/7] - in /kylin/site: ./ blog/ cn/docs21/ cn/docs21/install/ cn/docs21/tutorial/ docs21/ docs21/gettingstarted/ docs21/howto/ docs21/install/ docs21/tutorial/ images/tutorial/2.
Modified: kylin/site/docs21/howto/howto_backup_metadata.html URL: http://svn.apache.org/viewvc/kylin/site/docs21/howto/howto_backup_metadata.html?rev=1806793=1806792=1806793=diff == --- kylin/site/docs21/howto/howto_backup_metadata.html (original) +++ kylin/site/docs21/howto/howto_backup_metadata.html Thu Aug 31 14:01:54 2017 @@ -335,11 +335,7 @@ - - - - - FAQ + Technical Concepts @@ -359,7 +355,11 @@ - Events and Conferences + + + + + Terminology @@ -375,7 +375,13 @@ - Community Best Practices + + + + + + + FAQ @@ -395,11 +401,7 @@ - - - - - Terminology + Events and Conferences @@ -415,9 +417,7 @@ - - - Technical Concepts + Community Best Practices @@ -678,6 +678,8 @@ + + @@ -1054,7 +1056,53 @@ - Kylin Cube Creation + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Web Interface @@ -1132,11 +1180,7 @@ - - - - - Kylin Cube Build and Job Monitoring + Cube Wizard @@ -1218,11 +1262,7 @@ - - - - - Build Cube with Spark (beta) + Cube Build and Job Monitoring @@ -1296,7 +1336,7 @@ - Kylin Cube Permission + Cube Permission @@ -1382,6 +1422,54 @@ + Build Cube with Spark (beta) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -1418,11 +1506,29 @@ - Kylin Web Interface + Cube Build Tuning + + + + + + + + +Integration + + + + + + + + + @@ -1520,9 +1626,21 @@ + Kylin ODBC Driver + + + + + + + + + + + @@ -1532,7 +1650,15 @@ - Tableau 8 + + + + + + + + + Kylin JDBC Driver @@ -1648,9 +1774,7 @@ - - - Tableau 9 + Tableau 8 @@ -1754,7 +1878,23 @@ - MS Excel and Power BI + + + + + + + + + + + + + + + + + Tableau 9 @@ -1854,7 +1994,13 @@ - Kylin ODBC Driver + + + + + + + MS Excel and Power BI @@ -1944,7 +2090,15 @@ - Connect from Apache Flink + + + + + + + + + MicroStrategy @@ -2054,7 +2208,9 @@ - Connect from SQuirreL + + + SQuirreL @@ -2138,7 +2294,13 @@ - Cube Build Tuning Step by Step + + + + + + + Apache Flink @@ -2230,7 +2392,7 @@ - Connect from Apache Hue + Apache Hue @@ -2276,24 +2438,6 @@ - Build Cube with RESTful API - - - - - - - - - - - - - - - - - @@ -2314,19 +2458,7 @@ - - - - - - - - - - - - - Use RESTful API in Javascript + Use RESTful API @@ -2354,27 +2486,7 @@ - - - - - - - - - - - - - - - - - - - - - Use RESTful API + Build Cube with API @@ -2504,7 +2616,7 @@ - Cleanup Storage (HDFS & HBase) + Cleanup Storage @@ -2536,7 +2648,7 @@ - Use JDBC Driver + Kylin JDBC Driver @@ -2612,7 +2724,7 @@ - Enable Security with LDAP and SSO + Secure with LDAP and SSO @@ -2656,7 +2768,7 @@ - Use Beeline for Hive Commands + Use Beeline for Hive @@ -2696,7 +2808,7 @@ - How to Update HBase Coprocessor + Update Coprocessor Modified: kylin/site/docs21/howto/howto_build_cube_with_restapi.html URL:
svn commit: r1806793 [4/7] - in /kylin/site: ./ blog/ cn/docs21/ cn/docs21/install/ cn/docs21/tutorial/ docs21/ docs21/gettingstarted/ docs21/howto/ docs21/install/ docs21/tutorial/ images/tutorial/2.
Modified: kylin/site/docs21/install/hadoop_env.html URL: http://svn.apache.org/viewvc/kylin/site/docs21/install/hadoop_env.html?rev=1806793=1806792=1806793=diff == --- kylin/site/docs21/install/hadoop_env.html (original) +++ kylin/site/docs21/install/hadoop_env.html Thu Aug 31 14:01:54 2017 @@ -335,11 +335,7 @@ - - - - - FAQ + Technical Concepts @@ -359,7 +355,11 @@ - Events and Conferences + + + + + Terminology @@ -375,7 +375,13 @@ - Community Best Practices + + + + + + + FAQ @@ -395,11 +401,7 @@ - - - - - Terminology + Events and Conferences @@ -415,9 +417,7 @@ - - - Technical Concepts + Community Best Practices @@ -678,6 +678,8 @@ + + @@ -1054,7 +1056,53 @@ - Kylin Cube Creation + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Web Interface @@ -1132,11 +1180,7 @@ - - - - - Kylin Cube Build and Job Monitoring + Cube Wizard @@ -1218,11 +1262,7 @@ - - - - - Build Cube with Spark (beta) + Cube Build and Job Monitoring @@ -1296,7 +1336,7 @@ - Kylin Cube Permission + Cube Permission @@ -1382,6 +1422,54 @@ + Build Cube with Spark (beta) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -1418,11 +1506,29 @@ - Kylin Web Interface + Cube Build Tuning + + + + + + + + +Integration + + + + + + + + + @@ -1520,9 +1626,21 @@ + Kylin ODBC Driver + + + + + + + + + + + @@ -1532,7 +1650,15 @@ - Tableau 8 + + + + + + + + + Kylin JDBC Driver @@ -1648,9 +1774,7 @@ - - - Tableau 9 + Tableau 8 @@ -1754,7 +1878,23 @@ - MS Excel and Power BI + + + + + + + + + + + + + + + + + Tableau 9 @@ -1854,7 +1994,13 @@ - Kylin ODBC Driver + + + + + + + MS Excel and Power BI @@ -1944,7 +2090,15 @@ - Connect from Apache Flink + + + + + + + + + MicroStrategy @@ -2054,7 +2208,9 @@ - Connect from SQuirreL + + + SQuirreL @@ -2138,7 +2294,13 @@ - Cube Build Tuning Step by Step + + + + + + + Apache Flink @@ -2230,7 +2392,7 @@ - Connect from Apache Hue + Apache Hue @@ -2276,24 +2438,6 @@ - Build Cube with RESTful API - - - - - - - - - - - - - - - - - @@ -2314,19 +2458,7 @@ - - - - - - - - - - - - - Use RESTful API in Javascript + Use RESTful API @@ -2354,27 +2486,7 @@ - - - - - - - - - - - - - - - - - - - - - Use RESTful API + Build Cube with API @@ -2504,7 +2616,7 @@ - Cleanup Storage (HDFS & HBase) + Cleanup Storage @@ -2536,7 +2648,7 @@ - Use JDBC Driver + Kylin JDBC Driver @@ -2612,7 +2724,7 @@ - Enable Security with LDAP and SSO + Secure with LDAP and SSO @@ -2656,7 +2768,7 @@ - Use Beeline for Hive Commands + Use Beeline for Hive @@ -2696,7 +2808,7 @@ - How to Update HBase Coprocessor + Update Coprocessor Modified: kylin/site/docs21/install/index.html URL: http://svn.apache.org/viewvc/kylin/site/docs21/install/index.html?rev=1806793=1806792=1806793=diff
svn commit: r1806793 [7/7] - in /kylin/site: ./ blog/ cn/docs21/ cn/docs21/install/ cn/docs21/tutorial/ docs21/ docs21/gettingstarted/ docs21/howto/ docs21/install/ docs21/tutorial/ images/tutorial/2.
Modified: kylin/site/docs21/tutorial/tableau_91.html URL: http://svn.apache.org/viewvc/kylin/site/docs21/tutorial/tableau_91.html?rev=1806793=1806792=1806793=diff == --- kylin/site/docs21/tutorial/tableau_91.html (original) +++ kylin/site/docs21/tutorial/tableau_91.html Thu Aug 31 14:01:54 2017 @@ -335,11 +335,7 @@ - - - - - FAQ + Technical Concepts @@ -359,7 +355,11 @@ - Events and Conferences + + + + + Terminology @@ -375,7 +375,13 @@ - Community Best Practices + + + + + + + FAQ @@ -395,11 +401,7 @@ - - - - - Terminology + Events and Conferences @@ -415,9 +417,7 @@ - - - Technical Concepts + Community Best Practices @@ -678,6 +678,8 @@ + + @@ -1054,7 +1056,53 @@ - Kylin Cube Creation + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Web Interface @@ -1132,11 +1180,7 @@ - - - - - Kylin Cube Build and Job Monitoring + Cube Wizard @@ -1218,11 +1262,7 @@ - - - - - Build Cube with Spark (beta) + Cube Build and Job Monitoring @@ -1296,7 +1336,7 @@ - Kylin Cube Permission + Cube Permission @@ -1382,6 +1422,54 @@ + Build Cube with Spark (beta) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -1418,11 +1506,29 @@ - Kylin Web Interface + Cube Build Tuning + + + + + + + + +Integration + + + + + + + + + @@ -1520,9 +1626,21 @@ + Kylin ODBC Driver + + + + + + + + + + + @@ -1532,7 +1650,15 @@ - Tableau 8 + + + + + + + + + Kylin JDBC Driver @@ -1648,9 +1774,7 @@ - - - Tableau 9 + Tableau 8 @@ -1754,7 +1878,23 @@ - MS Excel and Power BI + + + + + + + + + + + + + + + + + Tableau 9 @@ -1854,7 +1994,13 @@ - Kylin ODBC Driver + + + + + + + MS Excel and Power BI @@ -1944,7 +2090,15 @@ - Connect from Apache Flink + + + + + + + + + MicroStrategy @@ -2054,7 +2208,9 @@ - Connect from SQuirreL + + + SQuirreL @@ -2138,7 +2294,13 @@ - Cube Build Tuning Step by Step + + + + + + + Apache Flink @@ -2230,7 +2392,7 @@ - Connect from Apache Hue + Apache Hue @@ -2276,24 +2438,6 @@ - Build Cube with RESTful API - - - - - - - - - - - - - - - - - @@ -2314,19 +2458,7 @@ - - - - - - - - - - - - - Use RESTful API in Javascript + Use RESTful API @@ -2354,27 +2486,7 @@ - - - - - - - - - - - - - - - - - - - - - Use RESTful API + Build Cube with API @@ -2504,7 +2616,7 @@ - Cleanup Storage (HDFS & HBase) + Cleanup Storage @@ -2536,7 +2648,7 @@ - Use JDBC Driver + Kylin JDBC Driver @@ -2612,7 +2724,7 @@ - Enable Security with LDAP and SSO + Secure with LDAP and SSO @@ -2656,7 +2768,7 @@ - Use Beeline for Hive Commands + Use Beeline for Hive @@ -2696,7 +2808,7 @@ - How to Update HBase Coprocessor + Update Coprocessor Modified: kylin/site/docs21/tutorial/web.html URL: http://svn.apache.org/viewvc/kylin/site/docs21/tutorial/web.html?rev=1806793=1806792=1806793=diff
svn commit: r1806793 [1/7] - in /kylin/site: ./ blog/ cn/docs21/ cn/docs21/install/ cn/docs21/tutorial/ docs21/ docs21/gettingstarted/ docs21/howto/ docs21/install/ docs21/tutorial/ images/tutorial/2.
Author: lidong Date: Thu Aug 31 14:01:54 2017 New Revision: 1806793 URL: http://svn.apache.org/viewvc?rev=1806793=rev Log: Update doc layout Added: kylin/site/docs21/tutorial/microstrategy.html kylin/site/images/tutorial/2.1/MicroStrategy/ kylin/site/images/tutorial/2.1/MicroStrategy/0.png (with props) kylin/site/images/tutorial/2.1/MicroStrategy/1.png (with props) kylin/site/images/tutorial/2.1/MicroStrategy/10.png (with props) kylin/site/images/tutorial/2.1/MicroStrategy/2.png (with props) kylin/site/images/tutorial/2.1/MicroStrategy/3.png (with props) kylin/site/images/tutorial/2.1/MicroStrategy/4.png (with props) kylin/site/images/tutorial/2.1/MicroStrategy/5.png (with props) kylin/site/images/tutorial/2.1/MicroStrategy/6.png (with props) kylin/site/images/tutorial/2.1/MicroStrategy/7.png (with props) kylin/site/images/tutorial/2.1/MicroStrategy/8.png (with props) kylin/site/images/tutorial/2.1/MicroStrategy/9.png (with props) Modified: kylin/site/blog/index.html kylin/site/cn/docs21/index.html kylin/site/cn/docs21/install/index.html kylin/site/cn/docs21/install/manual_install_guide.html kylin/site/cn/docs21/tutorial/acl.html kylin/site/cn/docs21/tutorial/create_cube.html kylin/site/cn/docs21/tutorial/cube_build_job.html kylin/site/cn/docs21/tutorial/kylin_client_tool.html kylin/site/cn/docs21/tutorial/odbc.html kylin/site/cn/docs21/tutorial/powerbi.html kylin/site/cn/docs21/tutorial/query_pushdown.html kylin/site/cn/docs21/tutorial/tableau.html kylin/site/cn/docs21/tutorial/tableau_91.html kylin/site/cn/docs21/tutorial/web.html kylin/site/docs21/gettingstarted/best_practices.html kylin/site/docs21/gettingstarted/concepts.html kylin/site/docs21/gettingstarted/events.html kylin/site/docs21/gettingstarted/faq.html kylin/site/docs21/gettingstarted/terminology.html kylin/site/docs21/howto/howto_backup_metadata.html kylin/site/docs21/howto/howto_build_cube_with_restapi.html kylin/site/docs21/howto/howto_cleanup_storage.html kylin/site/docs21/howto/howto_jdbc.html kylin/site/docs21/howto/howto_ldap_and_sso.html kylin/site/docs21/howto/howto_optimize_build.html kylin/site/docs21/howto/howto_optimize_cubes.html kylin/site/docs21/howto/howto_update_coprocessor.html kylin/site/docs21/howto/howto_upgrade.html kylin/site/docs21/howto/howto_use_beeline.html kylin/site/docs21/howto/howto_use_distributed_scheduler.html kylin/site/docs21/howto/howto_use_restapi.html kylin/site/docs21/howto/howto_use_restapi_in_js.html kylin/site/docs21/index.html kylin/site/docs21/install/advance_settings.html kylin/site/docs21/install/hadoop_env.html kylin/site/docs21/install/index.html kylin/site/docs21/install/kylin_cluster.html kylin/site/docs21/install/kylin_docker.html kylin/site/docs21/release_notes.html kylin/site/docs21/tutorial/acl.html kylin/site/docs21/tutorial/create_cube.html kylin/site/docs21/tutorial/cube_build_job.html kylin/site/docs21/tutorial/cube_build_performance.html kylin/site/docs21/tutorial/cube_spark.html kylin/site/docs21/tutorial/cube_streaming.html kylin/site/docs21/tutorial/flink.html kylin/site/docs21/tutorial/hue.html kylin/site/docs21/tutorial/kylin_sample.html kylin/site/docs21/tutorial/odbc.html kylin/site/docs21/tutorial/powerbi.html kylin/site/docs21/tutorial/query_pushdown.html kylin/site/docs21/tutorial/squirrel.html kylin/site/docs21/tutorial/tableau.html kylin/site/docs21/tutorial/tableau_91.html kylin/site/docs21/tutorial/web.html kylin/site/feed.xml Modified: kylin/site/blog/index.html URL: http://svn.apache.org/viewvc/kylin/site/blog/index.html?rev=1806793=1806792=1806793=diff == --- kylin/site/blog/index.html (original) +++ kylin/site/blog/index.html Thu Aug 31 14:01:54 2017 @@ -301,13 +301,13 @@ - Apache Kylin v1.5.1 Release Announcementposted: Apr 12, 2016 + Apache Kylin v1.5.1 æ£å¼åå¸posted: Apr 12, 2016 - Apache Kylin v1.5.1 æ£å¼åå¸posted: Apr 12, 2016 + Apache Kylin v1.5.1 Release Announcementposted: Apr 12, 2016 @@ -379,13 +379,13 @@ - Apache Kylin v1.1 (incubating) Release Announcementposted: Oct 25, 2015 + Apache Kylin v1.1 (incubating) æ£å¼åå¸posted: Oct 25, 2015 - Apache Kylin v1.1 (incubating) æ£å¼åå¸posted: Oct 25, 2015 + Apache Kylin v1.1 (incubating) Release Announcementposted: Oct 25, 2015 Modified: kylin/site/cn/docs21/index.html URL:
svn commit: r1806793 [3/7] - in /kylin/site: ./ blog/ cn/docs21/ cn/docs21/install/ cn/docs21/tutorial/ docs21/ docs21/gettingstarted/ docs21/howto/ docs21/install/ docs21/tutorial/ images/tutorial/2.
Modified: kylin/site/docs21/howto/howto_upgrade.html URL: http://svn.apache.org/viewvc/kylin/site/docs21/howto/howto_upgrade.html?rev=1806793=1806792=1806793=diff == --- kylin/site/docs21/howto/howto_upgrade.html (original) +++ kylin/site/docs21/howto/howto_upgrade.html Thu Aug 31 14:01:54 2017 @@ -335,11 +335,7 @@ - - - - - FAQ + Technical Concepts @@ -359,7 +355,11 @@ - Events and Conferences + + + + + Terminology @@ -375,7 +375,13 @@ - Community Best Practices + + + + + + + FAQ @@ -395,11 +401,7 @@ - - - - - Terminology + Events and Conferences @@ -415,9 +417,7 @@ - - - Technical Concepts + Community Best Practices @@ -678,6 +678,8 @@ + + @@ -1054,7 +1056,53 @@ - Kylin Cube Creation + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Web Interface @@ -1132,11 +1180,7 @@ - - - - - Kylin Cube Build and Job Monitoring + Cube Wizard @@ -1218,11 +1262,7 @@ - - - - - Build Cube with Spark (beta) + Cube Build and Job Monitoring @@ -1296,7 +1336,7 @@ - Kylin Cube Permission + Cube Permission @@ -1382,6 +1422,54 @@ + Build Cube with Spark (beta) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -1418,11 +1506,29 @@ - Kylin Web Interface + Cube Build Tuning + + + + + + + + +Integration + + + + + + + + + @@ -1520,9 +1626,21 @@ + Kylin ODBC Driver + + + + + + + + + + + @@ -1532,7 +1650,15 @@ - Tableau 8 + + + + + + + + + Kylin JDBC Driver @@ -1648,9 +1774,7 @@ - - - Tableau 9 + Tableau 8 @@ -1754,7 +1878,23 @@ - MS Excel and Power BI + + + + + + + + + + + + + + + + + Tableau 9 @@ -1854,7 +1994,13 @@ - Kylin ODBC Driver + + + + + + + MS Excel and Power BI @@ -1944,7 +2090,15 @@ - Connect from Apache Flink + + + + + + + + + MicroStrategy @@ -2054,7 +2208,9 @@ - Connect from SQuirreL + + + SQuirreL @@ -2138,7 +2294,13 @@ - Cube Build Tuning Step by Step + + + + + + + Apache Flink @@ -2230,7 +2392,7 @@ - Connect from Apache Hue + Apache Hue @@ -2276,24 +2438,6 @@ - Build Cube with RESTful API - - - - - - - - - - - - - - - - - @@ -2314,19 +2458,7 @@ - - - - - - - - - - - - - Use RESTful API in Javascript + Use RESTful API @@ -2354,27 +2486,7 @@ - - - - - - - - - - - - - - - - - - - - - Use RESTful API + Build Cube with API @@ -2504,7 +2616,7 @@ - Cleanup Storage (HDFS & HBase) + Cleanup Storage @@ -2536,7 +2648,7 @@ - Use JDBC Driver + Kylin JDBC Driver @@ -2612,7 +2724,7 @@ - Enable Security with LDAP and SSO + Secure with LDAP and SSO @@ -2656,7 +2768,7 @@ - Use Beeline for Hive Commands + Use Beeline for Hive @@ -2696,7 +2808,7 @@ - How to Update HBase Coprocessor + Update Coprocessor Modified: kylin/site/docs21/howto/howto_use_beeline.html URL: http://svn.apache.org/viewvc/kylin/site/docs21/howto/howto_use_beeline.html?rev=1806793=1806792=1806793=diff
svn commit: r1806793 [5/7] - in /kylin/site: ./ blog/ cn/docs21/ cn/docs21/install/ cn/docs21/tutorial/ docs21/ docs21/gettingstarted/ docs21/howto/ docs21/install/ docs21/tutorial/ images/tutorial/2.
Modified: kylin/site/docs21/tutorial/cube_build_performance.html URL: http://svn.apache.org/viewvc/kylin/site/docs21/tutorial/cube_build_performance.html?rev=1806793=1806792=1806793=diff == --- kylin/site/docs21/tutorial/cube_build_performance.html (original) +++ kylin/site/docs21/tutorial/cube_build_performance.html Thu Aug 31 14:01:54 2017 @@ -41,7 +41,7 @@ - Apache Kylin | Cube Build Tuning Step by Step + Apache Kylin | Cube Build Tuning @@ -335,11 +335,7 @@ - - - - - FAQ + Technical Concepts @@ -359,7 +355,11 @@ - Events and Conferences + + + + + Terminology @@ -375,7 +375,13 @@ - Community Best Practices + + + + + + + FAQ @@ -395,11 +401,7 @@ - - - - - Terminology + Events and Conferences @@ -415,9 +417,7 @@ - - - Technical Concepts + Community Best Practices @@ -678,6 +678,8 @@ + + @@ -1054,7 +1056,53 @@ - Kylin Cube Creation + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Web Interface @@ -1132,11 +1180,7 @@ - - - - - Kylin Cube Build and Job Monitoring + Cube Wizard @@ -1218,11 +1262,7 @@ - - - - - Build Cube with Spark (beta) + Cube Build and Job Monitoring @@ -1296,7 +1336,7 @@ - Kylin Cube Permission + Cube Permission @@ -1382,6 +1422,54 @@ + Build Cube with Spark (beta) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -1418,11 +1506,29 @@ - Kylin Web Interface + Cube Build Tuning + + + + + + + + +Integration + + + + + + + + + @@ -1520,9 +1626,21 @@ + Kylin ODBC Driver + + + + + + + + + + + @@ -1532,7 +1650,15 @@ - Tableau 8 + + + + + + + + + Kylin JDBC Driver @@ -1648,9 +1774,7 @@ - - - Tableau 9 + Tableau 8 @@ -1754,7 +1878,23 @@ - MS Excel and Power BI + + + + + + + + + + + + + + + + + Tableau 9 @@ -1854,7 +1994,13 @@ - Kylin ODBC Driver + + + + + + + MS Excel and Power BI @@ -1944,7 +2090,15 @@ - Connect from Apache Flink + + + + + + + + + MicroStrategy @@ -2054,7 +2208,9 @@ - Connect from SQuirreL + + + SQuirreL @@ -2138,7 +2294,13 @@ - Cube Build Tuning Step by Step + + + + + + + Apache Flink @@ -2230,7 +2392,7 @@ - Connect from Apache Hue + Apache Hue @@ -2276,24 +2438,6 @@ - Build Cube with RESTful API - - - - - - - - - - - - - - - - - @@ -2314,19 +2458,7 @@ - - - - - - - - - - - - - Use RESTful API in Javascript + Use RESTful API @@ -2354,27 +2486,7 @@ - - - - - - - - - - - - - - - - - - - - - Use RESTful API + Build Cube with API @@ -2504,7 +2616,7 @@ - Cleanup Storage (HDFS & HBase) + Cleanup Storage @@ -2536,7 +2648,7 @@ - Use JDBC Driver + Kylin JDBC Driver @@ -2612,7 +2724,7 @@ - Enable Security with LDAP and SSO + Secure with LDAP and SSO @@ -2656,7 +2768,7 @@ - Use Beeline for Hive Commands + Use Beeline for Hive @@ -2696,7 +2808,7 @@ - How to Update HBase Coprocessor + Update Coprocessor @@ -2716,7 +2828,7
svn commit: r1806793 [6/7] - in /kylin/site: ./ blog/ cn/docs21/ cn/docs21/install/ cn/docs21/tutorial/ docs21/ docs21/gettingstarted/ docs21/howto/ docs21/install/ docs21/tutorial/ images/tutorial/2.
Added: kylin/site/docs21/tutorial/microstrategy.html URL: http://svn.apache.org/viewvc/kylin/site/docs21/tutorial/microstrategy.html?rev=1806793=auto == --- kylin/site/docs21/tutorial/microstrategy.html (added) +++ kylin/site/docs21/tutorial/microstrategy.html Thu Aug 31 14:01:54 2017 @@ -0,0 +1,2976 @@ + + + + + + + + + + + + Apache Kylin | MicroStrategy + + + + + + + + + + + + + + + + + + + + + + http://kylin.apache.org/docs21/tutorial/microstrategy.html;> + http://kylin.apache.org/feed.xml; /> + + + + (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){ + (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o), + m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) + })(window,document,'script','//www.google-analytics.com/analytics.js','ga'); + + //oringal tracker for kylin.io + ga('create', 'UA-55534813-1', 'auto'); + //new tracker for kylin.apache.org + ga('create', 'UA-55534813-2', 'auto', {'name':'toplevel'}); + + ga('send', 'pageview'); + ga('toplevel.send', 'pageview'); + + + + + + + + + + + + + + + Apache Kylin⢠Extreme OLAP Engine for Big Data + + + http://apache.org/foundation/contributing.html; title="Support Apache" style="margin-left: 150px;"> + https://www.apache.org/images/SupportApache-small.png; style="height: 150px; width: 150px;"> + + + + + + + + + + + + +Toggle navigation + + + + + + + + + + + Home + Docs + Download + Community + Development + Blog + ä¸æç + https://twitter.com/apachekylin; target="_blank" class="fa fa-twitter fa-lg" title="Twitter: @ApacheKylin" > + https://github.com/apache/kylin; target="_blank" class="fa fa-github-alt fa-lg" title="Github: apache/kylin" > + https://www.facebook.com/kylinio; target="_blank" class="fa fa-facebook fa-lg" title="Facebook: kylin.io" > + + + + + + + + + + + + + + +Getting Started + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Overview + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Release Notes + + + + + + + + + + + + + + + + + + Technical Concepts + + + + + + + + + + + + + + + + + + + + + + + + Terminology + + + + + + + + + + + + + + + + + + + + + + FAQ + + + + + + + + + + + + + + + + + + + + Events and Conferences + + + + + + + + + + + + + + + + Community Best Practices + + + + + + + + + + + + +Installation + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Installation Guide + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Hadoop Environment + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
[3/4] kylin git commit: minor changes to MicroStrategy tutorial
minor changes to MicroStrategy tutorial Signed-off-by: shaofengshiProject: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2291fcee Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2291fcee Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2291fcee Branch: refs/heads/document Commit: 2291fcee5b6f0b66c4782372b8ded976352125da Parents: c8126da Author: joannahe Authored: Thu Aug 31 17:26:39 2017 +0800 Committer: shaofengshi Committed: Thu Aug 31 17:29:12 2017 +0800 -- website/_docs21/tutorial/microstrategy.md | 22 +++ website/images/tutorial/2.1/MicroStrategy/0.png | Bin 0 -> 207801 bytes 2 files changed, 13 insertions(+), 9 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/2291fcee/website/_docs21/tutorial/microstrategy.md -- diff --git a/website/_docs21/tutorial/microstrategy.md b/website/_docs21/tutorial/microstrategy.md index 7c549eb..67ec4c3 100644 --- a/website/_docs21/tutorial/microstrategy.md +++ b/website/_docs21/tutorial/microstrategy.md @@ -1,31 +1,35 @@ --- layout: docs21 -title: MicroStrategy 10 +title: Integration with MicroStrategy 10.X categories: tutorial permalink: /docs21/tutorial/microstrategy.html --- -## Integration with MicroStrategy 10.X - ### Install ODBC Driver Refer to this guide: [Kylin ODBC Driver Tutorial](./odbc.html). -Please make sure to download and install Kylin ODBC Driver __v1.6__ or above. If you already installed ODBC Driver in your system, please uninstall it first. already installed ODBC Driver in your system, please uninstall it first. +Please make sure to download and install Kylin ODBC Driver __v1.6__ 64 bit or above. If you already installed ODBC Driver in your system, please uninstall it first. already installed ODBC Driver in your system, please uninstall it first. -### Connect to Kylin Server +The Kylin ODBC driver needs to be installed in the machine or virtual environment where your Microstrategy Intelligenec Server is installed. -Connect Kylin using ODBC driver: open your MicroStrategy Developer and connect to the project source where your are going to connect Kylin data source using a user account with administrative privilege. +###Create Local DSN -Once logged in, go to `Administration` -> `Configuration manager` -> `Database Instance`, create a new database instance with local ODBC connection that you created in the previous step. Under database connection type, please choose Generic DBMS. +Open your window ODBC Data Source Administrator (64bit) and create a system DSN that point to your kylin instance. + +![](/images/tutorial/2.1/MicroStrategy/0.png) ### Setting Database Instance -Depending on your business scenario, you may need to create a new project and set Kylin database instance as your primary database instance or if there is an existing project, set Kylin database instance as one of your primary or non-primary database instance. You can achieve this by right click on your project, and go to `project configuration` -> `database instance`. +Connect Kylin using ODBC driver: open your MicroStrategy Developer and connect to the project source where your are going to connect Kylin data source using a user account with administrative privilege. + +Once logged in, go to `Administration` -> `Configuration manager` -> `Database Instance`, create a new database instance with system DSN that you created in the previous step. Under database connection type, please choose Generic DBMS. ![](/images/tutorial/2.1/MicroStrategy/2.png) ![](/images/tutorial/2.1/MicroStrategy/1.png) +Depending on your business scenario, you may need to create a new project and set Kylin database instance as your primary database instance or if there is an existing project, set Kylin database instance as one of your primary or non-primary database instance. You can achieve this by right click on your project, and go to `project configuration` -> `database instance`. + ### Import Logical Table Open up your project, go to `schema` -> `warehouse catalog` to import the tables your need. @@ -77,4 +81,4 @@ Now you can start creating reports with Kylin as data source. 3. On the top menu choose `Tools` -> `show Advanced Settings`. 4. Go to `select/insert` -> `date format`. 5. Change the date format to follow date format in Kylin, for example '-mm-dd'. - 6. Restart MicroStrategy Intelligence Server so that change can be effective. \ No newline at end of file + 6. Restart MicroStrategy Intelligence Server so that change can be effective. \ No newline at end of file
[4/4] kylin git commit: Update doc layout
Update doc layout Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c1af8222 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c1af8222 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c1af8222 Branch: refs/heads/document Commit: c1af8222f1b7d8196a2065e1cf3adfc638730cb8 Parents: 2291fce Author: shaofengshiAuthored: Thu Aug 31 18:57:47 2017 +0800 Committer: shaofengshi Committed: Thu Aug 31 18:57:47 2017 +0800 -- website/_data/docs21.yml| 22 ++ .../howto/howto_build_cube_with_restapi.md | 2 +- website/_docs21/howto/howto_cleanup_storage.md | 2 +- website/_docs21/howto/howto_jdbc.md | 2 +- website/_docs21/howto/howto_ldap_and_sso.md | 2 +- .../_docs21/howto/howto_update_coprocessor.md | 2 +- website/_docs21/howto/howto_use_beeline.md | 2 +- website/_docs21/howto/howto_use_restapi.md | 44 website/_docs21/index.md| 16 +++ website/_docs21/tutorial/acl.md | 2 +- website/_docs21/tutorial/create_cube.md | 2 +- website/_docs21/tutorial/cube_build_job.md | 2 +- .../_docs21/tutorial/cube_build_performance.md | 2 +- website/_docs21/tutorial/flink.md | 2 +- website/_docs21/tutorial/hue.md | 2 +- website/_docs21/tutorial/microstrategy.md | 2 +- website/_docs21/tutorial/squirrel.md| 2 +- website/_docs21/tutorial/web.md | 2 +- 18 files changed, 80 insertions(+), 32 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/c1af8222/website/_data/docs21.yml -- diff --git a/website/_data/docs21.yml b/website/_data/docs21.yml index 3f3380f..90acd70 100644 --- a/website/_data/docs21.yml +++ b/website/_data/docs21.yml @@ -19,11 +19,11 @@ docs: - index - release_notes + - gettingstarted/concepts + - gettingstarted/terminology - gettingstarted/faq - gettingstarted/events - gettingstarted/best_practices - - gettingstarted/terminology - - gettingstarted/concepts - title: Installation docs: @@ -37,25 +37,29 @@ - title: Tutorial docs: - tutorial/kylin_sample + - tutorial/web - tutorial/create_cube - tutorial/cube_build_job - - tutorial/cube_spark - tutorial/acl - - tutorial/web + - tutorial/cube_spark + - tutorial/cube_build_performance + +- title: Integration + docs: + - tutorial/odbc + - howto/howto_jdbc - tutorial/tableau - tutorial/tableau_91 - tutorial/powerbi - - tutorial/odbc - - tutorial/flink + - tutorial/microstrategy - tutorial/squirrel - - tutorial/cube_build_performance + - tutorial/flink - tutorial/hue - title: How To docs: - - howto/howto_build_cube_with_restapi - - howto/howto_use_restapi_in_js - howto/howto_use_restapi + - howto/howto_build_cube_with_restapi - howto/howto_optimize_cubes - howto/howto_optimize_build - howto/howto_backup_metadata http://git-wip-us.apache.org/repos/asf/kylin/blob/c1af8222/website/_docs21/howto/howto_build_cube_with_restapi.md -- diff --git a/website/_docs21/howto/howto_build_cube_with_restapi.md b/website/_docs21/howto/howto_build_cube_with_restapi.md index 9092afe..8808b7b 100644 --- a/website/_docs21/howto/howto_build_cube_with_restapi.md +++ b/website/_docs21/howto/howto_build_cube_with_restapi.md @@ -1,6 +1,6 @@ --- layout: docs21 -title: Build Cube with RESTful API +title: Build Cube with API categories: howto permalink: /docs21/howto/howto_build_cube_with_restapi.html --- http://git-wip-us.apache.org/repos/asf/kylin/blob/c1af8222/website/_docs21/howto/howto_cleanup_storage.md -- diff --git a/website/_docs21/howto/howto_cleanup_storage.md b/website/_docs21/howto/howto_cleanup_storage.md index 323e138..f488040 100644 --- a/website/_docs21/howto/howto_cleanup_storage.md +++ b/website/_docs21/howto/howto_cleanup_storage.md @@ -1,6 +1,6 @@ --- layout: docs21 -title: Cleanup Storage (HDFS & HBase) +title: Cleanup Storage categories: howto permalink: /docs21/howto/howto_cleanup_storage.html --- http://git-wip-us.apache.org/repos/asf/kylin/blob/c1af8222/website/_docs21/howto/howto_jdbc.md -- diff --git a/website/_docs21/howto/howto_jdbc.md b/website/_docs21/howto/howto_jdbc.md index f2fd6f0..4885a84 100644 --- a/website/_docs21/howto/howto_jdbc.md +++ b/website/_docs21/howto/howto_jdbc.md @@ -1,6 +1,6 @@ --- layout: docs21 -title: Use JDBC Driver +title: Kylin JDBC Driver categories: howto
[2/4] kylin git commit: minor update
minor update Signed-off-by: shaofengshiProject: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c8126dac Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c8126dac Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c8126dac Branch: refs/heads/document Commit: c8126dac9e7a8868b9aa31fafb30a20b4b09286c Parents: 6555f6d Author: joannahe Authored: Thu Aug 31 17:03:53 2017 +0800 Committer: shaofengshi Committed: Thu Aug 31 17:11:02 2017 +0800 -- website/_docs21/tutorial/microstrategy.md | 18 +- 1 file changed, 9 insertions(+), 9 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/c8126dac/website/_docs21/tutorial/microstrategy.md -- diff --git a/website/_docs21/tutorial/microstrategy.md b/website/_docs21/tutorial/microstrategy.md index 8a7cc45..7c549eb 100644 --- a/website/_docs21/tutorial/microstrategy.md +++ b/website/_docs21/tutorial/microstrategy.md @@ -5,7 +5,7 @@ categories: tutorial permalink: /docs21/tutorial/microstrategy.html --- -## Integration from MicroStrategy 10.X +## Integration with MicroStrategy 10.X ### Install ODBC Driver @@ -18,15 +18,15 @@ Connect Kylin using ODBC driver: open your MicroStrategy Developer and connect t Once logged in, go to `Administration` -> `Configuration manager` -> `Database Instance`, create a new database instance with local ODBC connection that you created in the previous step. Under database connection type, please choose Generic DBMS. -![](/images/tutorial/2.1/MicroStrategy/1.png) +### Setting Database Instance -### Setting Database instance +Depending on your business scenario, you may need to create a new project and set Kylin database instance as your primary database instance or if there is an existing project, set Kylin database instance as one of your primary or non-primary database instance. You can achieve this by right click on your project, and go to `project configuration` -> `database instance`. -Depending on your business scenario, you may need to create a new project and set kylin database instance as your primary database instance or if there is an existing project, set kylin database instance as one of your primary or non-primary database instance. You can achieve this by right click on your project, and go to `project configuration` -> `database instance`. +![](/images/tutorial/2.1/MicroStrategy/2.png) ![](/images/tutorial/2.1/MicroStrategy/1.png) -### Import logical table +### Import Logical Table Open up your project, go to `schema` -> `warehouse catalog` to import the tables your need. @@ -46,13 +46,13 @@ Create Attribute, Facts and Metric objects ### Create a Simple Report -Now you can start creating reports with Kylin as Data source. +Now you can start creating reports with Kylin as data source. ![](/images/tutorial/2.1/MicroStrategy/9.png) ![](/images/tutorial/2.1/MicroStrategy/10.png) -### Best Practice for connecting MicroStrategy to Kylin datasource +### Best Practice for Connecting MicroStrategy to Kylin Data Source 1. Kylin does not work with multiple SQL passes at the moment, so it is recommended to set up your report intermediate table type as derived, you can change this setting at report level using `Data`-> `VLDB property`-> `Tables`-> `Intermediate Table Type` @@ -64,7 +64,7 @@ Now you can start creating reports with Kylin as Data source. â Reports with custom groups -3. Dimension named with kylin keywords will cause sql to error out. You may find kylin keywords here, it is recommended to avoid naming the column name as kylin keywords, especially when you use MicroStrategy as the front-end BI tool, as far as I am concern there is no setting MicroStrategy that can escape the keyword. [https://calcite.apache.org/docs/reference.html#keywords](https://calcite.apache.org/docs/reference.html#keywords) +3. Dimension named with Kylin keywords will cause sql to error out. You may find Kylin keywords here, it is recommended to avoid naming the column name as Kylin keywords, especially when you use MicroStrategy as the front-end BI tool, as far as we know there is no setting in MicroStrategy that can escape the keyword. [https://calcite.apache.org/docs/reference.html#keywords](https://calcite.apache.org/docs/reference.html#keywords) 4. If underlying Kylin data model has left join from fact table to lookup table, In order for Microstrategy to also generate the same left join in sql, please follow below MicroStrategy TN to modify VLDB property: @@ -76,5 +76,5 @@ Now you can start creating reports with Kylin as Data source. 2. Then right click on the database,
[1/4] kylin git commit: add tutorial for MicroStrategy
Repository: kylin Updated Branches: refs/heads/document 08ab05ec4 -> c1af8222f add tutorial for MicroStrategy Signed-off-by: shaofengshiProject: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6555f6d2 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6555f6d2 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6555f6d2 Branch: refs/heads/document Commit: 6555f6d2628f190cfd3442732875b83120e88312 Parents: 08ab05e Author: joannahe Authored: Thu Aug 31 16:30:58 2017 +0800 Committer: shaofengshi Committed: Thu Aug 31 16:35:00 2017 +0800 -- website/_docs21/tutorial/microstrategy.md | 80 +++ website/images/tutorial/2.1/MicroStrategy/1.png | Bin 0 -> 207356 bytes .../images/tutorial/2.1/MicroStrategy/10.png| Bin 0 -> 185351 bytes website/images/tutorial/2.1/MicroStrategy/2.png | Bin 0 -> 154312 bytes website/images/tutorial/2.1/MicroStrategy/3.png | Bin 0 -> 345509 bytes website/images/tutorial/2.1/MicroStrategy/4.png | Bin 0 -> 187026 bytes website/images/tutorial/2.1/MicroStrategy/5.png | Bin 0 -> 158694 bytes website/images/tutorial/2.1/MicroStrategy/6.png | Bin 0 -> 143233 bytes website/images/tutorial/2.1/MicroStrategy/7.png | Bin 0 -> 125824 bytes website/images/tutorial/2.1/MicroStrategy/8.png | Bin 0 -> 283974 bytes website/images/tutorial/2.1/MicroStrategy/9.png | Bin 0 -> 357212 bytes 11 files changed, 80 insertions(+) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/6555f6d2/website/_docs21/tutorial/microstrategy.md -- diff --git a/website/_docs21/tutorial/microstrategy.md b/website/_docs21/tutorial/microstrategy.md new file mode 100644 index 000..8a7cc45 --- /dev/null +++ b/website/_docs21/tutorial/microstrategy.md @@ -0,0 +1,80 @@ +--- +layout: docs21 +title: MicroStrategy 10 +categories: tutorial +permalink: /docs21/tutorial/microstrategy.html +--- + +## Integration from MicroStrategy 10.X + +### Install ODBC Driver + +Refer to this guide: [Kylin ODBC Driver Tutorial](./odbc.html). +Please make sure to download and install Kylin ODBC Driver __v1.6__ or above. If you already installed ODBC Driver in your system, please uninstall it first. already installed ODBC Driver in your system, please uninstall it first. + +### Connect to Kylin Server + +Connect Kylin using ODBC driver: open your MicroStrategy Developer and connect to the project source where your are going to connect Kylin data source using a user account with administrative privilege. + +Once logged in, go to `Administration` -> `Configuration manager` -> `Database Instance`, create a new database instance with local ODBC connection that you created in the previous step. Under database connection type, please choose Generic DBMS. + +![](/images/tutorial/2.1/MicroStrategy/1.png) + +### Setting Database instance + +Depending on your business scenario, you may need to create a new project and set kylin database instance as your primary database instance or if there is an existing project, set kylin database instance as one of your primary or non-primary database instance. You can achieve this by right click on your project, and go to `project configuration` -> `database instance`. + +![](/images/tutorial/2.1/MicroStrategy/1.png) + +### Import logical table + +Open up your project, go to `schema` -> `warehouse catalog` to import the tables your need. + +![](/images/tutorial/2.1/MicroStrategy/4.png) + +### Building Schema and Public Objects + +Create Attribute, Facts and Metric objects + +![](/images/tutorial/2.1/MicroStrategy/5.png) + +![](/images/tutorial/2.1/MicroStrategy/6.png) + +![](/images/tutorial/2.1/MicroStrategy/7.png) + +![](/images/tutorial/2.1/MicroStrategy/8.png) + +### Create a Simple Report + +Now you can start creating reports with Kylin as Data source. + +![](/images/tutorial/2.1/MicroStrategy/9.png) + +![](/images/tutorial/2.1/MicroStrategy/10.png) + +### Best Practice for connecting MicroStrategy to Kylin datasource + +1. Kylin does not work with multiple SQL passes at the moment, so it is recommended to set up your report intermediate table type as derived, you can change this setting at report level using `Data`-> `VLDB property`-> `Tables`-> `Intermediate Table Type` + +2. Avoid using below functionality in MicroStrategy as it will generate multiple sql passes that can not be bypassed by VLDB property: + + â Creation of datamarts + + â Query partitioned tables + + â Reports with custom groups + +3. Dimension named with kylin keywords will cause sql to error out. You may find kylin keywords here, it is recommended to avoid naming the column name as kylin keywords, especially when you use
[3/4] kylin git commit: KYLIN-2814: fix connection may not be released when sync wipe cache
KYLIN-2814: fix connection may not be released when sync wipe cache Signed-off-by: Billy LiuProject: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5c1aab10 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5c1aab10 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5c1aab10 Branch: refs/heads/KYLIN-2814 Commit: 5c1aab101f3d5a3fb0cdc4c85c1c3f60274979fc Parents: 727920b Author: lichao Authored: Fri Aug 25 14:31:17 2017 +0800 Committer: Billy Liu Committed: Thu Aug 31 15:38:31 2017 +0800 -- .../java/org/apache/kylin/common/restclient/RestClient.java| 6 +- 1 file changed, 5 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/5c1aab10/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java -- diff --git a/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java b/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java index 13490cb..efd0438 100644 --- a/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java +++ b/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java @@ -121,8 +121,9 @@ public class RestClient { String url = baseUrl + "/cache/" + entity + "/" + cacheKey + "/" + event; HttpPut request = new HttpPut(url); +HttpResponse response = null; try { -HttpResponse response = client.execute(request); +response = client.execute(request); if (response.getStatusLine().getStatusCode() != 200) { String msg = EntityUtils.toString(response.getEntity()); @@ -131,6 +132,9 @@ public class RestClient { } catch (Exception ex) { throw new IOException(ex); } finally { +if(response != null){ +EntityUtils.consume(response.getEntity()); +} request.releaseConnection(); } }
[1/4] kylin git commit: KYLIN-2811, refine spark cubing [Forced Update!]
Repository: kylin Updated Branches: refs/heads/KYLIN-2814 abd13082b -> 3f80488e3 (forced update) KYLIN-2811, refine spark cubing Signed-off-by: Hongbin MaProject: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a1c234a9 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a1c234a9 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a1c234a9 Branch: refs/heads/KYLIN-2814 Commit: a1c234a9afbc5a30306f4275127649f980ab75bd Parents: 4316cfd Author: Cheng Wang Authored: Sun Aug 27 13:29:12 2017 +0800 Committer: Hongbin Ma Committed: Sun Aug 27 15:08:18 2017 +0800 -- .../org/apache/kylin/common/KylinConfig.java| 50 +-- .../metadata/model/ComputedColumnDesc.java | 10 +- .../kylin/engine/mr/common/BatchConstants.java | 1 + .../spark/SparkBatchCubingJobBuilder2.java | 1 + .../kylin/engine/spark/SparkCubingByLayer.java | 380 --- .../localmeta/cube_desc/ci_inner_join_cube.json | 2 +- .../kylin/rest/util/Log4jConfigListener.java| 1 + 7 files changed, 267 insertions(+), 178 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/a1c234a9/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java -- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java index a56e9b8..1d5e0ec 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java @@ -18,6 +18,15 @@ package org.apache.kylin.common; +import com.google.common.base.Preconditions; +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.kylin.common.restclient.RestClient; +import org.apache.kylin.common.util.ClassUtil; +import org.apache.kylin.common.util.OrderedProperties; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import java.io.BufferedReader; import java.io.File; import java.io.FileInputStream; @@ -34,26 +43,15 @@ import java.util.Enumeration; import java.util.Map; import java.util.Properties; -import org.apache.commons.io.IOUtils; -import org.apache.commons.lang.StringUtils; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.kylin.common.restclient.RestClient; -import org.apache.kylin.common.util.ClassUtil; -import org.apache.kylin.common.util.HadoopUtil; -import org.apache.kylin.common.util.OrderedProperties; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.google.common.base.Preconditions; - /** */ public class KylinConfig extends KylinConfigBase { private static final long serialVersionUID = 1L; private static final Logger logger = LoggerFactory.getLogger(KylinConfig.class); -/** Kylin properties file name */ +/** + * Kylin properties file name + */ public static final String KYLIN_CONF_PROPERTIES_FILE = "kylin.properties"; public static final String KYLIN_CONF = "KYLIN_CONF"; @@ -62,7 +60,7 @@ public class KylinConfig extends KylinConfigBase { // thread-local instances, will override SYS_ENV_INSTANCE private static transient ThreadLocal THREAD_ENV_INSTANCE = new ThreadLocal<>(); - + static { /* * Make Calcite to work with Unicode. @@ -121,9 +119,6 @@ public class KylinConfig extends KylinConfigBase { } private static UriType decideUriType(String metaUri) { -if (metaUri.indexOf("@hdfs") > 0) { -return UriType.HDFS_FILE; -} try { File file = new File(metaUri); @@ -163,23 +158,6 @@ public class KylinConfig extends KylinConfigBase { */ UriType uriType = decideUriType(uri); -if (uriType == UriType.HDFS_FILE) { -KylinConfig config; -FileSystem fs; -int cut = uri.indexOf('@'); -String realHdfsPath = uri.substring(0, cut) + "/" + KYLIN_CONF_PROPERTIES_FILE; -try { -config = new KylinConfig(); -fs = HadoopUtil.getFileSystem(realHdfsPath); -InputStream is = fs.open(new Path(realHdfsPath)); -Properties prop = streamToProps(is); -config.reloadKylinConfig(prop); -} catch (IOException e) { -throw new RuntimeException(e); -} -return config; -} - if (uriType == UriType.LOCAL_FOLDER) { KylinConfig config = new KylinConfig(); config.setMetadataUrl(uri); @@ -241,7 +219,7 @@ public class KylinConfig extends KylinConfigBase {
[4/4] kylin git commit: KYLIN-2814 close response for RestClient's request
KYLIN-2814 close response for RestClient's request Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3f80488e Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3f80488e Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3f80488e Branch: refs/heads/KYLIN-2814 Commit: 3f80488e35d6c9b971833d64257040b938b689fd Parents: 5c1aab1 Author: Billy LiuAuthored: Sun Aug 27 12:47:33 2017 +0800 Committer: Billy Liu Committed: Thu Aug 31 15:38:31 2017 +0800 -- .../kylin/common/restclient/RestClient.java | 112 +++-- .../kylin/restclient/ITRestClientTest.java | 120 ++- 2 files changed, 135 insertions(+), 97 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/3f80488e/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java -- diff --git a/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java b/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java index efd0438..6a89b9d 100644 --- a/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java +++ b/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java @@ -37,7 +37,7 @@ import org.apache.http.client.methods.HttpGet; import org.apache.http.client.methods.HttpPost; import org.apache.http.client.methods.HttpPut; import org.apache.http.client.methods.HttpRequestBase; -import org.apache.http.entity.StringEntity; +import org.apache.http.client.utils.HttpClientUtils; import org.apache.http.impl.client.BasicCredentialsProvider; import org.apache.http.impl.client.DefaultHttpClient; import org.apache.http.params.BasicHttpParams; @@ -129,12 +129,8 @@ public class RestClient { String msg = EntityUtils.toString(response.getEntity()); throw new IOException("Invalid response " + response.getStatusLine().getStatusCode() + " with cache wipe url " + url + "\n" + msg); } -} catch (Exception ex) { -throw new IOException(ex); } finally { -if(response != null){ -EntityUtils.consume(response.getEntity()); -} +HttpClientUtils.closeQuietly(response); request.releaseConnection(); } } @@ -142,8 +138,10 @@ public class RestClient { public String getKylinProperties() throws IOException { String url = baseUrl + "/admin/config"; HttpGet request = new HttpGet(url); + +HttpResponse response = null; try { -HttpResponse response = client.execute(request); +response = client.execute(request); String msg = EntityUtils.toString(response.getEntity()); Map map = JsonUtil.readValueAsMap(msg); msg = map.get("config"); @@ -152,83 +150,27 @@ public class RestClient { throw new IOException("Invalid response " + response.getStatusLine().getStatusCode() + " with cache wipe url " + url + "\n" + msg); return msg; } finally { +HttpClientUtils.closeQuietly(response); request.releaseConnection(); } } -public boolean enableCache() throws IOException { -return setCache(true); -} - -public boolean disableCache() throws IOException { -return setCache(false); -} - -public boolean buildCube(String cubeName, long startTime, long endTime, String buildType) throws Exception { -String url = baseUrl + "/cubes/" + cubeName + "/build"; -HttpPut put = newPut(url); -HashMap paraMap = new HashMap (); -paraMap.put("startTime", startTime + ""); -paraMap.put("endTime", endTime + ""); -paraMap.put("buildType", buildType); -String jsonMsg = new ObjectMapper().writeValueAsString(paraMap); -put.setEntity(new StringEntity(jsonMsg, "UTF-8")); -HttpResponse response = client.execute(put); -String result = getContent(response); -if (response.getStatusLine().getStatusCode() != 200) { -throw new IOException("Invalid response " + response.getStatusLine().getStatusCode() + " with build cube url " + url + "\n" + jsonMsg); -} else { -return true; -} -} - -public boolean disableCube(String cubeName) throws Exception { -return changeCubeStatus(baseUrl + "/cubes/" + cubeName + "/disable"); -} - -public boolean enableCube(String cubeName) throws Exception { -return changeCubeStatus(baseUrl + "/cubes/" + cubeName + "/enable"); -} - -public boolean purgeCube(String cubeName) throws
[2/4] kylin git commit: KYLIN-2811, fix NPE
KYLIN-2811, fix NPE Signed-off-by: Hongbin MaProject: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/727920b4 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/727920b4 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/727920b4 Branch: refs/heads/KYLIN-2814 Commit: 727920b4d3642aaa3657d90b7f3dce7dcdd68fe2 Parents: a1c234a Author: Cheng Wang Authored: Mon Aug 28 12:47:58 2017 +0800 Committer: Hongbin Ma Committed: Mon Aug 28 14:41:26 2017 +0800 -- .../kylin/engine/spark/SparkCubingByLayer.java | 43 +++- 1 file changed, 24 insertions(+), 19 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/727920b4/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java -- diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java index a3a6ad0..a03e238 100644 --- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java +++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java @@ -100,7 +100,6 @@ public class SparkCubingByLayer extends AbstractApplication implements Serializa .withDescription("Configuration Path").create("confPath"); private Options options; -private static String metaUrl; public SparkCubingByLayer() { options = new Options(); @@ -117,7 +116,7 @@ public class SparkCubingByLayer extends AbstractApplication implements Serializa return options; } -public static KylinConfig getKylinConfigForExecutor() { +public static KylinConfig getKylinConfigForExecutor(String metaUrl) { File file = new File(SparkFiles.get(KylinConfig.KYLIN_CONF_PROPERTIES_FILE)); String confPath = file.getParentFile().getAbsolutePath(); System.setProperty(KylinConfig.KYLIN_CONF, confPath); @@ -128,7 +127,7 @@ public class SparkCubingByLayer extends AbstractApplication implements Serializa @Override protected void execute(OptionsHelper optionsHelper) throws Exception { -metaUrl = optionsHelper.getOptionValue(OPTION_META_URL); +String metaUrl = optionsHelper.getOptionValue(OPTION_META_URL); String hiveTable = optionsHelper.getOptionValue(OPTION_INPUT_TABLE); String cubeName = optionsHelper.getOptionValue(OPTION_CUBE_NAME); String segmentId = optionsHelper.getOptionValue(OPTION_SEGMENT_ID); @@ -178,17 +177,17 @@ public class SparkCubingByLayer extends AbstractApplication implements Serializa // encode with dimension encoding, transform to RDD final JavaPairRDD encodedBaseRDD = intermediateTable.javaRDD() -.mapToPair(new EncodeBaseCuboid(cubeName, segmentId)); +.mapToPair(new EncodeBaseCuboid(cubeName, segmentId, metaUrl)); Long totalCount = 0L; if (envConfig.isSparkSanityCheckEnabled()) { totalCount = encodedBaseRDD.count(); } -final BaseCuboidReducerFunction2 baseCuboidReducerFunction = new BaseCuboidReducerFunction2(cubeName); +final BaseCuboidReducerFunction2 baseCuboidReducerFunction = new BaseCuboidReducerFunction2(cubeName, metaUrl); BaseCuboidReducerFunction2 reducerFunction2 = baseCuboidReducerFunction; if (allNormalMeasure == false) { -reducerFunction2 = new CuboidReducerFunction2(cubeName, needAggr); +reducerFunction2 = new CuboidReducerFunction2(cubeName, metaUrl, needAggr); } final int totalLevels = cubeDesc.getBuildLevel(); @@ -202,18 +201,18 @@ public class SparkCubingByLayer extends AbstractApplication implements Serializa Configuration confOverwrite = new Configuration(sc.hadoopConfiguration()); confOverwrite.set("dfs.replication", "2"); // cuboid intermediate files, replication=2 -saveToHDFS(allRDDs[0], cubeName, cubeSegment, outputPath, 0, confOverwrite); +saveToHDFS(allRDDs[0], cubeName, metaUrl, cubeSegment, outputPath, 0, confOverwrite); // aggregate to ND cuboids for (level = 1; level <= totalLevels; level++) { long levelRddSize = SizeEstimator.estimate(allRDDs[level - 1]) / (1024 * 1024); partition = estimateRDDPartitionNum(level, cubeStatsReader, envConfig, (int) levelRddSize); -allRDDs[level] = allRDDs[level - 1].flatMapToPair(new CuboidFlatMap(cubeName, segmentId)) +allRDDs[level] = allRDDs[level - 1].flatMapToPair(new CuboidFlatMap(cubeName, segmentId,
kylin git commit: APACHE-KYLIN-2735: Introduce an option to make job scheduler consider job priority
Repository: kylin Updated Branches: refs/heads/yaho-cube-planner 0b12df776 -> d216eba3e APACHE-KYLIN-2735: Introduce an option to make job scheduler consider job priority Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d216eba3 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d216eba3 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d216eba3 Branch: refs/heads/yaho-cube-planner Commit: d216eba3e10af99cb58b9ca45582d2bf5c8a0a37 Parents: 0b12df7 Author: ZhongAuthored: Thu Aug 31 14:56:19 2017 +0800 Committer: Zhong Committed: Thu Aug 31 14:56:19 2017 +0800 -- .../apache/kylin/common/KylinConfigBase.java| 8 ++ .../kylin/job/engine/JobEngineConfig.java | 14 +++ .../kylin/job/execution/AbstractExecutable.java | 9 ++ .../job/execution/CheckpointExecutable.java | 7 ++ .../job/execution/DefaultChainedExecutable.java | 7 ++ .../job/impl/threadpool/DefaultScheduler.java | 111 ++- .../org/apache/kylin/engine/mr/CubingJob.java | 19 +++- 7 files changed, 171 insertions(+), 4 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/d216eba3/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java -- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index d66d7ce..53bb2ad 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -503,6 +503,14 @@ abstract public class KylinConfigBase implements Serializable { return Integer.parseInt(getOptional("kylin.job.scheduler.default", "0")); } +public boolean getSchedulerPriorityConsidered() { +return Boolean.parseBoolean(getOptional("kylin.job.scheduler.priority-considered", "false")); +} + +public Integer getSchedulerPriorityBarFetchFromQueue() { +return Integer.parseInt(getOptional("kylin.job.scheduler.priority-bar-fetch-from-queue", "20")); +} + public Integer getSchedulerPollIntervalSecond() { return Integer.parseInt(getOptional("kylin.job.scheduler.poll-interval-second", "30")); } http://git-wip-us.apache.org/repos/asf/kylin/blob/d216eba3/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java -- diff --git a/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java b/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java index 6890557..9ba602f 100644 --- a/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java +++ b/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java @@ -105,6 +105,20 @@ public class JobEngineConfig { return config; } +/** + * @return if consider job priority when scheduling jobs + * */ +public boolean getJobPriorityConsidered() { +return config.getSchedulerPriorityConsidered(); +} + +/** + * @return the priority bar for fetching jobs from job priority queue + */ +public int getFetchQueuePriorityBar() { +return config.getSchedulerPriorityBarFetchFromQueue(); +} + public String getHdfsWorkingDirectory() { return config.getHdfsWorkingDirectory(); } http://git-wip-us.apache.org/repos/asf/kylin/blob/d216eba3/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java -- diff --git a/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java b/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java index 30b6421..a37cdc9 100644 --- a/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java +++ b/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java @@ -44,6 +44,8 @@ import com.google.common.collect.Maps; */ public abstract class AbstractExecutable implements Executable, Idempotent { +public static final Integer DEFAULT_PRIORITY = 10; + protected static final String SUBMITTER = "submitter"; protected static final String NOTIFY_LIST = "notify_list"; protected static final String START_TIME = "startTime"; @@ -389,6 +391,13 @@ public abstract class AbstractExecutable implements Executable, Idempotent { return output.getState() == ExecutableState.READY; } +/** + * The larger the value, the higher priority + * */ +public int getDefaultPriority() { +return DEFAULT_PRIORITY; +} + /*
[kylin] Git Push Summary [forced push!] [Forced Update!]
Repository: kylin Updated Branches: refs/heads/yaho-cube-planner 214ddfefa -> 0b12df776 (forced update)
kylin git commit: APACHE-KYLIN-2735: Introduce an option to make job scheduler consider job priority
Repository: kylin Updated Branches: refs/heads/yaho-cube-planner 0b12df776 -> 214ddfefa APACHE-KYLIN-2735: Introduce an option to make job scheduler consider job priority Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/214ddfef Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/214ddfef Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/214ddfef Branch: refs/heads/yaho-cube-planner Commit: 214ddfefaeddc1b08a4963f300cf2e2277603d0b Parents: 0b12df7 Author: ZhongAuthored: Thu Aug 31 14:01:26 2017 +0800 Committer: Zhong Committed: Thu Aug 31 14:01:26 2017 +0800 -- .../apache/kylin/common/KylinConfigBase.java| 4 + .../kylin/job/engine/JobEngineConfig.java | 7 ++ .../kylin/job/execution/AbstractExecutable.java | 9 ++ .../job/execution/CheckpointExecutable.java | 7 ++ .../job/execution/DefaultChainedExecutable.java | 7 ++ .../job/impl/threadpool/DefaultScheduler.java | 112 ++- .../org/apache/kylin/engine/mr/CubingJob.java | 19 +++- 7 files changed, 161 insertions(+), 4 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/214ddfef/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java -- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index d66d7ce..1419a48 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -503,6 +503,10 @@ abstract public class KylinConfigBase implements Serializable { return Integer.parseInt(getOptional("kylin.job.scheduler.default", "0")); } +public boolean getSchedulerPriorityConsidered() { +return Boolean.parseBoolean(getOptional("kylin.job.scheduler.priority-considered", "false")); +} + public Integer getSchedulerPollIntervalSecond() { return Integer.parseInt(getOptional("kylin.job.scheduler.poll-interval-second", "30")); } http://git-wip-us.apache.org/repos/asf/kylin/blob/214ddfef/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java -- diff --git a/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java b/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java index 6890557..baa2aaa 100644 --- a/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java +++ b/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java @@ -105,6 +105,13 @@ public class JobEngineConfig { return config; } +/** + * @return if consider job priority when scheduling jobs + * */ +public boolean getJobPriorityConsidered() { +return config.getSchedulerPriorityConsidered(); +} + public String getHdfsWorkingDirectory() { return config.getHdfsWorkingDirectory(); } http://git-wip-us.apache.org/repos/asf/kylin/blob/214ddfef/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java -- diff --git a/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java b/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java index 30b6421..a37cdc9 100644 --- a/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java +++ b/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java @@ -44,6 +44,8 @@ import com.google.common.collect.Maps; */ public abstract class AbstractExecutable implements Executable, Idempotent { +public static final Integer DEFAULT_PRIORITY = 10; + protected static final String SUBMITTER = "submitter"; protected static final String NOTIFY_LIST = "notify_list"; protected static final String START_TIME = "startTime"; @@ -389,6 +391,13 @@ public abstract class AbstractExecutable implements Executable, Idempotent { return output.getState() == ExecutableState.READY; } +/** + * The larger the value, the higher priority + * */ +public int getDefaultPriority() { +return DEFAULT_PRIORITY; +} + /* * discarded is triggered by JobService, the Scheduler is not awake of that * http://git-wip-us.apache.org/repos/asf/kylin/blob/214ddfef/core-job/src/main/java/org/apache/kylin/job/execution/CheckpointExecutable.java -- diff --git a/core-job/src/main/java/org/apache/kylin/job/execution/CheckpointExecutable.java