KYLIN-2240 Add a toggle to ignore all cube signature inconsistency temporally
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/80018874 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/80018874 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/80018874 Branch: refs/heads/yang21-hbase1.x Commit: 80018874c26d17f57bd288654996d3b6508d8294 Parents: 076c77a Author: Hongbin Ma <mahong...@apache.org> Authored: Thu Dec 1 11:35:34 2016 +0800 Committer: Hongbin Ma <mahong...@apache.org> Committed: Thu Dec 1 11:35:34 2016 +0800 ---------------------------------------------------------------------- .../apache/kylin/common/KylinConfigBase.java | 39 ++++++--- .../org/apache/kylin/cube/model/CubeDesc.java | 90 +++++++++++--------- 2 files changed, 76 insertions(+), 53 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/80018874/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index a91e42b..f35f969 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -18,6 +18,13 @@ package org.apache.kylin.common; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; +import org.apache.commons.lang.StringUtils; +import org.apache.kylin.common.util.CliCommandExecutor; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import java.io.File; import java.io.IOException; import java.io.Serializable; @@ -28,14 +35,6 @@ import java.util.SortedSet; import java.util.regex.Matcher; import java.util.regex.Pattern; -import org.apache.commons.lang.StringUtils; -import org.apache.kylin.common.util.CliCommandExecutor; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.google.common.collect.Maps; -import com.google.common.collect.Sets; - /** * An abstract class to encapsulate access to a set of 'properties'. * Subclass can override methods in this class to extend the content of the 'properties', @@ -174,19 +173,25 @@ abstract public class KylinConfigBase implements Serializable { setProperty("kylin.storage.url", storageUrl); } - /** was for route to hive, not used any more */ + /** + * was for route to hive, not used any more + */ @Deprecated public String getHiveUrl() { return getOptional("hive.url", ""); } - /** was for route to hive, not used any more */ + /** + * was for route to hive, not used any more + */ @Deprecated public String getHiveUser() { return getOptional("hive.user", ""); } - /** was for route to hive, not used any more */ + /** + * was for route to hive, not used any more + */ @Deprecated public String getHivePassword() { return getOptional("hive.password", ""); @@ -202,7 +207,7 @@ abstract public class KylinConfigBase implements Serializable { public String[] getRealizationProviders() { return getOptionalStringArray("kylin.realization.providers", // - new String[] { "org.apache.kylin.cube.CubeManager", "org.apache.kylin.storage.hybrid.HybridManager" }); + new String[]{"org.apache.kylin.cube.CubeManager", "org.apache.kylin.storage.hybrid.HybridManager"}); } public CliCommandExecutor getCliCommandExecutor() throws IOException { @@ -427,6 +432,10 @@ abstract public class KylinConfigBase implements Serializable { return Integer.parseInt(getOptional("kylin.cube.algorithm.auto.mapper.limit", "500")); } + public boolean isIgnoreCubeSignatureInconsistency() { + return Boolean.parseBoolean(getOptional("kylin.cube.ignore-signature-inconsistency", "false")); + } + @Deprecated public int getCubeAggrGroupMaxSize() { return Integer.parseInt(getOptional("kylin.cube.aggrgroup.max.size", "12")); @@ -480,10 +489,11 @@ abstract public class KylinConfigBase implements Serializable { public float getCubeVisitTimeoutTimes() { return Float.parseFloat(getOptional("kylin.query.cube.visit.timeout.times", "1")); } - + public int getDerivedInThreshold() { return Integer.parseInt(getOptional("kylin.query.filter.derived_in.max", "20")); } + public int getBadQueryStackTraceDepth() { return Integer.parseInt(getOptional("kylin.query.badquery.stacktrace.depth", "10")); } @@ -569,7 +579,7 @@ abstract public class KylinConfigBase implements Serializable { } public int[] getQueryMetricsPercentilesIntervals() { - String[] dft = { "60", "300", "3600" }; + String[] dft = {"60", "300", "3600"}; return getOptionalIntArray("kylin.query.metrics.percentiles.intervals", dft); } @@ -591,6 +601,7 @@ abstract public class KylinConfigBase implements Serializable { /** * HBase region cut size, in GB + * * @return */ public float getKylinHBaseRegionCut() { http://git-wip-us.apache.org/repos/asf/kylin/blob/80018874/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java index c914ebb..00bfbc9 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java @@ -18,27 +18,16 @@ package org.apache.kylin.cube.model; -import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.base.Preconditions.checkNotNull; -import static com.google.common.base.Preconditions.checkState; - -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Set; -import java.util.TreeSet; - -import javax.annotation.Nullable; - +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.google.common.base.Function; +import com.google.common.collect.Collections2; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; import org.apache.commons.codec.binary.Base64; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.ArrayUtils; @@ -67,16 +56,25 @@ import org.apache.kylin.metadata.model.TblColRef; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.fasterxml.jackson.annotation.JsonAutoDetect; -import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.google.common.base.Function; -import com.google.common.collect.Collections2; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; -import com.google.common.collect.Sets; +import javax.annotation.Nullable; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.TreeSet; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.common.base.Preconditions.checkState; /** */ @@ -199,7 +197,7 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware { } /** - * @return dimension columns excluding derived + * @return dimension columns excluding derived */ public List<TblColRef> listDimensionColumnsExcludingDerived(boolean alsoExcludeExtendedCol) { List<TblColRef> result = new ArrayList<TblColRef>(); @@ -467,11 +465,17 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware { /** * this method is to prevent malicious metadata change by checking the saved signature * with the calculated signature. - * + * <p> * if you're comparing two cube descs, prefer to use consistentWith() + * * @return */ public boolean checkSignature() { + if (this.getConfig().isIgnoreCubeSignatureInconsistency()) { + logger.info("Skip checking cube signature"); + return true; + } + if (KylinVersion.getCurrentVersion().isCompatibleWith(new KylinVersion(getVersion())) && !KylinVersion.getCurrentVersion().isSignatureCompatibleWith(new KylinVersion(getVersion()))) { logger.info("checkSignature on {} is skipped as the its version is {} (not signature compatible but compatible) ", getName(), getVersion()); return true; @@ -752,7 +756,7 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware { int find = ArrayUtils.indexOf(dimColArray, fk[i]); if (find >= 0) { TblColRef derivedCol = initDimensionColRef(pk[i]); - initDerivedMap(new TblColRef[] { dimColArray[find] }, DeriveType.PK_FK, dim, new TblColRef[] { derivedCol }, null); + initDerivedMap(new TblColRef[]{dimColArray[find]}, DeriveType.PK_FK, dim, new TblColRef[]{derivedCol}, null); } } /** disable this code as we don't need fk be derived from pk @@ -782,7 +786,7 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware { extra[i] = ""; } } - return new String[][] { cols, extra }; + return new String[][]{cols, extra}; } private void initDerivedMap(TblColRef[] hostCols, DeriveType type, DimensionDesc dimension, TblColRef[] derivedCols, String[] extra) { @@ -1011,7 +1015,9 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware { this.partitionDateEnd = partitionDateEnd; } - /** Get columns that have dictionary */ + /** + * Get columns that have dictionary + */ public Set<TblColRef> getAllColumnsHaveDictionary() { Set<TblColRef> result = Sets.newLinkedHashSet(); @@ -1040,7 +1046,9 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware { return result; } - /** Get columns that need dictionary built on it. Note a column could reuse dictionary of another column. */ + /** + * Get columns that need dictionary built on it. Note a column could reuse dictionary of another column. + */ public Set<TblColRef> getAllColumnsNeedDictionaryBuilt() { Set<TblColRef> result = getAllColumnsHaveDictionary(); @@ -1057,7 +1065,9 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware { return result; } - /** A column may reuse dictionary of another column, find the dict column, return same col if there's no reuse column*/ + /** + * A column may reuse dictionary of another column, find the dict column, return same col if there's no reuse column + */ public TblColRef getDictionaryReuseColumn(TblColRef col) { if (dictionaries == null) { return col; @@ -1070,7 +1080,9 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware { return col; } - /** Get a column which can be used in distributing the source table */ + /** + * Get a column which can be used in distributing the source table + */ public TblColRef getDistributedByColumn() { Set<TblColRef> shardBy = getShardByColumns(); if (shardBy != null && shardBy.size() > 0) {
