KYLIN-2030 enhancement
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/46284bdc Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/46284bdc Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/46284bdc Branch: refs/heads/master Commit: 46284bdca211d59f4a3274f5579dc8fe8a48cc6e Parents: 1429da5 Author: Hongbin Ma <mahong...@apache.org> Authored: Sat Oct 8 16:14:26 2016 +0800 Committer: Hongbin Ma <mahong...@apache.org> Committed: Thu Oct 27 08:30:13 2016 +0800 ---------------------------------------------------------------------- .../kylin/cube/CubeCapabilityChecker.java | 43 +++++++++++++++----- .../org/apache/kylin/cube/model/CubeDesc.java | 6 +-- .../kylin/metadata/model/FunctionDesc.java | 1 + .../test/resources/query/sql_lookup/query06.sql | 1 + .../test/resources/query/sql_lookup/query07.sql | 1 + 5 files changed, 39 insertions(+), 13 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/46284bdc/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java index c8460f0..ee21b1c 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java @@ -25,18 +25,22 @@ import java.util.List; import java.util.Set; import org.apache.commons.lang.StringUtils; +import org.apache.kylin.common.KylinConfig; import org.apache.kylin.cube.model.CubeDesc; import org.apache.kylin.measure.MeasureType; import org.apache.kylin.measure.basic.BasicMeasureType; +import org.apache.kylin.metadata.MetadataManager; import org.apache.kylin.metadata.filter.UDF.MassInTupleFilter; +import org.apache.kylin.metadata.model.ColumnDesc; import org.apache.kylin.metadata.model.FunctionDesc; import org.apache.kylin.metadata.model.IStorageAware; import org.apache.kylin.metadata.model.MeasureDesc; import org.apache.kylin.metadata.model.ParameterDesc; +import org.apache.kylin.metadata.model.TableDesc; import org.apache.kylin.metadata.model.TblColRef; import org.apache.kylin.metadata.realization.CapabilityResult; -import org.apache.kylin.metadata.realization.CapabilityResult.CapabilityInfluence; import org.apache.kylin.metadata.realization.SQLDigest; +import org.apache.kylin.metadata.realization.CapabilityResult.CapabilityInfluence; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -66,20 +70,39 @@ public class CubeCapabilityChecker { Collection<FunctionDesc> unmatchedAggregations = unmatchedAggregations(aggrFunctions, cube); // try custom measure types + // in RAW query, unmatchedDimensions and unmatchedAggregations will null, so can't chose RAW cube well! + // if (!unmatchedDimensions.isEmpty() || !unmatchedAggregations.isEmpty()) { tryCustomMeasureTypes(unmatchedDimensions, unmatchedAggregations, digest, cube, result); + // } - // try dimension-as-measure - if (!unmatchedAggregations.isEmpty()) { - if (cube.getDescriptor().getFactTable().equals(digest.factTable)) { + //more tricks + if (cube.getDescriptor().getFactTable().equals(digest.factTable)) { + //for query-on-facttable + //1. dimension as measure + + if (!unmatchedAggregations.isEmpty()) { tryDimensionAsMeasures(unmatchedAggregations, digest, cube, result, cube.getDescriptor().listDimensionColumnsIncludingDerived()); - } else { - //deal with query on lookup table, like https://issues.apache.org/jira/browse/KYLIN-2030 - if (cube.getSegments().get(0).getSnapshots().containsKey(digest.factTable)) { - Set<TblColRef> dimCols = Sets.newHashSet(cube.getDataModelDesc().findFirstTable(digest.factTable).getColumns()); + } + } else { + //for non query-on-facttable + if (cube.getSegments().get(0).getSnapshots().containsKey(digest.factTable)) { + TableDesc tableDesc = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv()).getTableDesc(digest.factTable); + Set<TblColRef> dimCols = Sets.newHashSet(); + for (ColumnDesc columnDesc : tableDesc.getColumns()) { + dimCols.add(columnDesc.getRef()); + } + + //1. dimension as measure, like max(cal_dt) or count( distinct col) from lookup + if (!unmatchedAggregations.isEmpty()) { tryDimensionAsMeasures(unmatchedAggregations, digest, cube, result, dimCols); - } else { - logger.info("Skip tryDimensionAsMeasures because current cube {} does not touch lookup table {} at all", cube.getName(), digest.factTable); } + + //2. more "dimensions" contributed by snapshot + if (!unmatchedDimensions.isEmpty()) { + unmatchedDimensions.removeAll(dimCols); + } + } else { + logger.info("cube {} does not touch lookup table {} at all", cube.getName(), digest.factTable); } } http://git-wip-us.apache.org/repos/asf/kylin/blob/46284bdc/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java index 0c0b4f3..159042d 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java @@ -33,9 +33,9 @@ import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; -import java.util.Map.Entry; import java.util.Set; import java.util.TreeSet; +import java.util.Map.Entry; import javax.annotation.Nullable; @@ -67,9 +67,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.fasterxml.jackson.annotation.JsonAutoDetect; -import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; import com.fasterxml.jackson.core.JsonProcessingException; import com.google.common.base.Function; import com.google.common.collect.Collections2; @@ -674,7 +674,7 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware { private boolean hasSingle(ArrayList<Set<String>> dimsList) { boolean hasSingle = false; for (Set<String> dims : dimsList) { - if (dims.size() < 2) + if (dims.size() == 1) hasSingle = true; } return hasSingle; http://git-wip-us.apache.org/repos/asf/kylin/blob/46284bdc/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java index e7d5186..ae7f805 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java @@ -102,6 +102,7 @@ public class FunctionDesc { } public MeasureType<?> getMeasureType() { + //like max(cal_dt) if (isDimensionAsMetric && !isCountDistinct()) { return null; } http://git-wip-us.apache.org/repos/asf/kylin/blob/46284bdc/kylin-it/src/test/resources/query/sql_lookup/query06.sql ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/resources/query/sql_lookup/query06.sql b/kylin-it/src/test/resources/query/sql_lookup/query06.sql new file mode 100644 index 0000000..e2b8469 --- /dev/null +++ b/kylin-it/src/test/resources/query/sql_lookup/query06.sql @@ -0,0 +1 @@ +select max(cal_dt) as x from edw.test_cal_dt \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/46284bdc/kylin-it/src/test/resources/query/sql_lookup/query07.sql ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/resources/query/sql_lookup/query07.sql b/kylin-it/src/test/resources/query/sql_lookup/query07.sql new file mode 100644 index 0000000..6be3439 --- /dev/null +++ b/kylin-it/src/test/resources/query/sql_lookup/query07.sql @@ -0,0 +1 @@ +select site_id,count(*) as y,count(DISTINCT site_name) as x from edw.test_sites group by site_id
