KYLIN-1495 Metadata upgrade tool from 1.0~1.3 to 1.5
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2512fb6f Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2512fb6f Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2512fb6f Branch: refs/heads/master Commit: 2512fb6fb33aad0dae2105c0479417694419cacf Parents: f1ecd8e Author: Hongbin Ma <[email protected]> Authored: Tue Mar 15 17:34:32 2016 +0800 Committer: Hongbin Ma <[email protected]> Committed: Wed Mar 16 23:42:01 2016 +0800 ---------------------------------------------------------------------- build/bin/upgrade_metadata_v_1_5_0.sh | 46 + .../org/apache/kylin/common/KylinConfig.java | 5 +- .../org/apache/kylin/common/KylinVersion.java | 3 +- .../kylin/cube/cli/CubeSignatureRefresher.java | 135 +++ .../org/apache/kylin/cube/model/CubeDesc.java | 17 +- .../cube/model/v1_4_0/CubeBuildTypeEnum.java | 39 + .../kylin/cube/model/v1_4_0/CubeDesc.java | 854 +++++++++++++++++++ .../kylin/cube/model/v1_4_0/DimensionDesc.java | 239 ++++++ .../cube/model/v1_4_0/HBaseColumnDesc.java | 138 +++ .../model/v1_4_0/HBaseColumnFamilyDesc.java | 59 ++ .../cube/model/v1_4_0/HBaseMappingDesc.java | 96 +++ .../kylin/cube/model/v1_4_0/HierarchyDesc.java | 68 ++ .../kylin/cube/model/v1_4_0/RowKeyColDesc.java | 92 ++ .../kylin/cube/model/v1_4_0/RowKeyDesc.java | 301 +++++++ .../kylin/cube/model/v3/CubeBuildTypeEnum.java | 39 - .../apache/kylin/cube/model/v3/CubeDesc.java | 825 ------------------ .../kylin/cube/model/v3/DimensionDesc.java | 239 ------ .../kylin/cube/model/v3/HBaseColumnDesc.java | 138 --- .../cube/model/v3/HBaseColumnFamilyDesc.java | 58 -- .../kylin/cube/model/v3/HBaseMappingDesc.java | 96 --- .../kylin/cube/model/v3/HierarchyDesc.java | 68 -- .../kylin/cube/model/v3/RowKeyColDesc.java | 92 -- .../apache/kylin/cube/model/v3/RowKeyDesc.java | 295 ------- .../cube/upgrade/CubeDescSignatureUpdate.java | 130 --- .../upgrade/V1_5_0/CubeDescUpgrade_v_1_5_0.java | 284 ++++++ .../V1_5_0/CubeMetadataUpgrade_v_1_5_0.java | 150 ++++ .../cube/upgrade/V3/CubeDescUpgraderV3.java | 287 ------- .../cube/upgrade/V3/CubeMetadataUpgradeV3.java | 180 ---- .../upgrade/common/CubeMetadataUpgrade.java | 146 ++++ .../common/MetadataVersionRefresher.java | 86 ++ .../entry/CubeMetadataUpgradeEntry_v_1_5_0.java | 46 + .../v1_4_0/CubeMetadataUpgrade_v_1_4_0.java | 181 ++++ .../apache/kylin/metadata/MetadataManager.java | 16 +- .../kylin/metadata/model/DataModelDesc.java | 10 +- .../kylin/metadata/model/DimensionDesc.java | 74 -- .../metadata/model/ModelDimensionDesc.java | 74 ++ .../kylin/metadata/model/PartitionDesc.java | 4 +- examples/test_case_data/localmeta/cube/ssb.json | 1 - .../cube/test_kylin_cube_with_slr_empty.json | 1 - .../test_case_data/localmeta/cube_desc/ssb.json | 1 - .../kylin/invertedindex/model/IIDesc.java | 20 +- .../hbase/steps/SandboxMetastoreCLI.java | 4 +- 42 files changed, 3084 insertions(+), 2553 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/2512fb6f/build/bin/upgrade_metadata_v_1_5_0.sh ---------------------------------------------------------------------- diff --git a/build/bin/upgrade_metadata_v_1_5_0.sh b/build/bin/upgrade_metadata_v_1_5_0.sh new file mode 100644 index 0000000..b60204b --- /dev/null +++ b/build/bin/upgrade_metadata_v_1_5_0.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# the script will upgrade Kylin 1.0 ~ 1.3 compatible metadata store to 1.5 compatible metadata store +# the approach is 1) upgrade to 1.4 compatible 2) upgrade from 1.4 compatible to 1.5 compatible + +dir=$(dirname ${0}) +source ${dir}/check-env.sh + +# start command +if [ "$#" -ne 1 ] +then + echo "usage: upgrade_metadata_v_1_5_0.sh current_metadata_store_dump_path" + exit 1 +fi + + +echo "=====Upgrade Cube metadata to 1.4 compatible =====" +$KYLIN_HOME/bin/kylin.sh org.apache.kylin.cube.upgrade.v1_4_0.CubeMetadataUpgrade_v_1_4_0 $1 + +echo "=====Upgrade Cube metadata to 1.5 compatible =====" +$KYLIN_HOME/bin/kylin.sh org.apache.kylin.cube.upgrade.V1_5_0.CubeMetadataUpgrade_v_1_5_0 $1 + +echo "=====Refresh all cubes' signature =====" +$KYLIN_HOME/bin/kylin.sh org.apache.kylin.cube.cli.CubeSignatureRefresher + +echo "======Deploy coprocessor=======" +$KYLIN_HOME/bin/kylin.sh org.apache.kylin.storage.hbase.util.DeployCoprocessorCLI $KYLIN_HOME/lib/kylin-coprocessor-2.0-SNAPSHOT.jar all + +echo "==============End==============" http://git-wip-us.apache.org/repos/asf/kylin/blob/2512fb6f/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java index 81f5827..13b232c 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java @@ -46,6 +46,10 @@ public class KylinConfig extends KylinConfigBase { private static final Logger logger = LoggerFactory.getLogger(KylinConfig.class); + static { + Log4jConfigurer.initLogger(); + } + /** Kylin properties file name */ public static final String KYLIN_CONF_PROPERTIES_FILE = "kylin.properties"; public static final String KYLIN_CONF = "KYLIN_CONF"; @@ -226,7 +230,6 @@ public class KylinConfig extends KylinConfigBase { * $KYLIN_CONF/kylin.properties 2. Check the $KYLIN_HOME/conf/kylin.properties */ private static KylinConfig loadKylinConfig() { - Log4jConfigurer.initLogger(); InputStream is = getKylinPropertiesAsInputSteam(); if (is == null) { http://git-wip-us.apache.org/repos/asf/kylin/blob/2512fb6f/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java b/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java index 74faa49..df7fc30 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java @@ -16,7 +16,6 @@ */ package org.apache.kylin.common; - public class KylinVersion { /** * Require MANUAL updating kylin version per ANY upgrading. @@ -30,7 +29,7 @@ public class KylinVersion { * * @return current kylin version in String */ - public static String getCurrentVersion(){ + public static String getCurrentVersion() { return CURRENT_KYLIN_VERSION; } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/2512fb6f/core-cube/src/main/java/org/apache/kylin/cube/cli/CubeSignatureRefresher.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cli/CubeSignatureRefresher.java b/core-cube/src/main/java/org/apache/kylin/cube/cli/CubeSignatureRefresher.java new file mode 100644 index 0000000..2e2b500 --- /dev/null +++ b/core-cube/src/main/java/org/apache/kylin/cube/cli/CubeSignatureRefresher.java @@ -0,0 +1,135 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kylin.cube.cli; + +import java.util.List; + +import org.apache.commons.lang.ArrayUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.kylin.common.KylinConfig; +import org.apache.kylin.common.persistence.ResourceStore; +import org.apache.kylin.cube.CubeDescManager; +import org.apache.kylin.cube.CubeManager; +import org.apache.kylin.cube.model.CubeDesc; +import org.apache.kylin.metadata.MetadataManager; +import org.apache.kylin.metadata.project.ProjectManager; + +import com.google.common.collect.Lists; + +/** + * used to bulk refresh the cube's signature in metadata store. + * won't be useful unless something went wrong. + */ +public class CubeSignatureRefresher { + private static final Log logger = LogFactory.getLog(CubeSignatureRefresher.class); + private KylinConfig config = null; + private ResourceStore store; + private String[] cubeNames; + private List<String> updatedResources = Lists.newArrayList(); + private List<String> errorMsgs = Lists.newArrayList(); + + public CubeSignatureRefresher(String[] cubes) { + config = KylinConfig.getInstanceFromEnv(); + store = ResourceStore.getStore(config); + cubeNames = cubes; + } + + public void update() { + logger.info("Reloading Cube Metadata from store: " + store.getReadableResourcePath(ResourceStore.CUBE_DESC_RESOURCE_ROOT)); + CubeDescManager cubeDescManager = CubeDescManager.getInstance(config); + List<CubeDesc> cubeDescs; + if (ArrayUtils.isEmpty(cubeNames)) { + cubeDescs = cubeDescManager.listAllDesc(); + } else { + String[] names = cubeNames[0].split(","); + if (ArrayUtils.isEmpty(names)) + return; + cubeDescs = Lists.newArrayListWithCapacity(names.length); + for (String name : names) { + cubeDescs.add(cubeDescManager.getCubeDesc(name)); + } + } + for (CubeDesc cubeDesc : cubeDescs) { + updateCubeDesc(cubeDesc); + } + + verify(); + } + + private void verify() { + MetadataManager.getInstance(config).reload(); + CubeDescManager.clearCache(); + CubeDescManager.getInstance(config); + CubeManager.getInstance(config); + ProjectManager.getInstance(config); + } + + public List<String> getErrorMsgs() { + return errorMsgs; + } + + private void updateCubeDesc(CubeDesc cubeDesc) { + try { + String calculatedSign = cubeDesc.calculateSignature(); + if (cubeDesc.getSignature() == null || (!cubeDesc.getSignature().equals(calculatedSign)) ){ + cubeDesc.setSignature(calculatedSign); + store.putResource(cubeDesc.getResourcePath(), cubeDesc, CubeDescManager.CUBE_DESC_SERIALIZER); + updatedResources.add(cubeDesc.getResourcePath()); + } + } catch (Exception e) { + logger.error("error", e); + errorMsgs.add("Update CubeDesc[" + cubeDesc.getName() + "] failed: " + e.getLocalizedMessage()); + } + } + + public static void main(String args[]) { + if (args != null && args.length > 1) { + System.out.println("Usage: java CubeDescSignatureUpdate [Cubes]; e.g, cube1,cube2 "); + return; + } + + CubeSignatureRefresher metadataUpgrade = new CubeSignatureRefresher(args); + metadataUpgrade.update(); + + logger.info("================================================================="); + logger.info("Run CubeDescSignatureUpdate completed;"); + + if (!metadataUpgrade.updatedResources.isEmpty()) { + logger.info("Following resources are updated successfully:"); + for (String s : metadataUpgrade.updatedResources) { + logger.info(s); + } + } else { + logger.warn("No resource updated."); + } + + if (!metadataUpgrade.errorMsgs.isEmpty()) { + logger.info("Here are the error/warning messages, you may need to check:"); + for (String s : metadataUpgrade.errorMsgs) { + logger.warn(s); + } + } else { + logger.info("No error or warning messages; The update succeeds."); + } + + logger.info("================================================================="); + } + +} http://git-wip-us.apache.org/repos/asf/kylin/blob/2512fb6f/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java index 49f70f8..105aa82 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java @@ -40,6 +40,7 @@ import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.ArrayUtils; import org.apache.commons.lang.StringUtils; import org.apache.kylin.common.KylinConfig; +import org.apache.kylin.common.KylinVersion; import org.apache.kylin.common.persistence.ResourceStore; import org.apache.kylin.common.persistence.RootPersistentEntity; import org.apache.kylin.common.util.Array; @@ -438,6 +439,11 @@ public class CubeDesc extends RootPersistentEntity { } public boolean checkSignature() { + if (!KylinVersion.getCurrentVersion().equals(getVersion())) { + logger.info("checkSignature on {} is skipped as the its version is {} (not latest but compatible) ", getName(), getVersion()); + return true; + } + if (StringUtils.isBlank(getSignature())) { return true; } @@ -454,11 +460,14 @@ public class CubeDesc extends RootPersistentEntity { StringBuilder sigString = new StringBuilder(); sigString.append(this.name).append("|")// .append(JsonUtil.writeValueAsString(this.modelName)).append("|")// + .append(JsonUtil.writeValueAsString(this.nullStrings)).append("|")// .append(JsonUtil.writeValueAsString(this.dimensions)).append("|")// .append(JsonUtil.writeValueAsString(this.measures)).append("|")// .append(JsonUtil.writeValueAsString(this.rowkey)).append("|")// .append(JsonUtil.writeValueAsString(this.aggregationGroups)).append("|")// - .append(JsonUtil.writeValueAsString(this.hbaseMapping)); + .append(JsonUtil.writeValueAsString(this.hbaseMapping)).append("|")// + .append(JsonUtil.writeValueAsString(this.engineType)).append("|")// + .append(JsonUtil.writeValueAsString(this.storageType)).append("|"); byte[] signature = md.digest(sigString.toString().toLowerCase().getBytes()); String ret = new String(Base64.encodeBase64(signature)); @@ -573,12 +582,12 @@ public class CubeDesc extends RootPersistentEntity { if (CollectionUtils.containsAny(mandatoryDims, jointDims)) { logger.warn("Aggregation group " + index + " mandatory dims overlap with joint dims"); } - + if (CollectionUtils.containsAny(hierarchyDims, jointDims)) { logger.error("Aggregation group " + index + " hierarchy dims overlap with joint dims"); throw new IllegalStateException("Aggregation group " + index + " hierarchy dims overlap with joint dims"); } - + if (hasSingle(hierarchyDimsList)) { logger.error("Aggregation group " + index + " require at least 2 dims in a hierarchy"); throw new IllegalStateException("Aggregation group " + index + " require at least 2 dims in a hierarchy"); @@ -587,7 +596,7 @@ public class CubeDesc extends RootPersistentEntity { logger.error("Aggregation group " + index + " require at least 2 dims in a joint"); throw new IllegalStateException("Aggregation group " + index + " require at least 2 dims in a joint"); } - + if (hasOverlap(hierarchyDimsList, hierarchyDims)) { logger.error("Aggregation group " + index + " a dim exist in more than one hierarchy"); throw new IllegalStateException("Aggregation group " + index + " a dim exist in more than one hierarchy"); http://git-wip-us.apache.org/repos/asf/kylin/blob/2512fb6f/core-cube/src/main/java/org/apache/kylin/cube/model/v1_4_0/CubeBuildTypeEnum.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/v1_4_0/CubeBuildTypeEnum.java b/core-cube/src/main/java/org/apache/kylin/cube/model/v1_4_0/CubeBuildTypeEnum.java new file mode 100644 index 0000000..50123b1 --- /dev/null +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/v1_4_0/CubeBuildTypeEnum.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kylin.cube.model.v1_4_0; + +/** + * @author xduo + * + */ +public enum CubeBuildTypeEnum { + /** + * rebuild a segment or incremental build + */ + BUILD, + /** + * merge segments + */ + MERGE, + + /** + * refresh segments + */ + REFRESH +} http://git-wip-us.apache.org/repos/asf/kylin/blob/2512fb6f/core-cube/src/main/java/org/apache/kylin/cube/model/v1_4_0/CubeDesc.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/v1_4_0/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/v1_4_0/CubeDesc.java new file mode 100644 index 0000000..8327a2a --- /dev/null +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/v1_4_0/CubeDesc.java @@ -0,0 +1,854 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kylin.cube.model.v1_4_0; + +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; + +import org.apache.commons.codec.binary.Base64; +import org.apache.commons.lang.ArrayUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.kylin.common.KylinConfig; +import org.apache.kylin.common.persistence.ResourceStore; +import org.apache.kylin.common.persistence.RootPersistentEntity; +import org.apache.kylin.common.util.Array; +import org.apache.kylin.common.util.CaseInsensitiveStringMap; +import org.apache.kylin.common.util.JsonUtil; +import org.apache.kylin.metadata.MetadataConstants; +import org.apache.kylin.metadata.MetadataManager; +import org.apache.kylin.metadata.model.ColumnDesc; +import org.apache.kylin.metadata.model.DataModelDesc; +import org.apache.kylin.metadata.model.FunctionDesc; +import org.apache.kylin.metadata.model.IEngineAware; +import org.apache.kylin.metadata.model.IStorageAware; +import org.apache.kylin.metadata.model.JoinDesc; +import org.apache.kylin.metadata.model.MeasureDesc; +import org.apache.kylin.metadata.model.TableDesc; +import org.apache.kylin.metadata.model.TblColRef; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; + +/** + */ +@SuppressWarnings("serial") +@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE) +public class CubeDesc extends RootPersistentEntity { + + public static enum DeriveType { + LOOKUP, PK_FK + } + + public static class DeriveInfo { + public DeriveType type; + public DimensionDesc dimension; + public TblColRef[] columns; + public boolean isOneToOne; // only used when ref from derived to host + + DeriveInfo(DeriveType type, DimensionDesc dimension, TblColRef[] columns, boolean isOneToOne) { + this.type = type; + this.dimension = dimension; + this.columns = columns; + this.isOneToOne = isOneToOne; + } + + @Override + public String toString() { + return "DeriveInfo [type=" + type + ", dimension=" + dimension + ", columns=" + Arrays.toString(columns) + ", isOneToOne=" + isOneToOne + "]"; + } + + } + + private KylinConfig config; + private DataModelDesc model; + + @JsonProperty("name") + private String name; + @JsonProperty("model_name") + private String modelName; + @JsonProperty("description") + private String description; + @JsonProperty("null_string") + private String[] nullStrings; + @JsonProperty("dimensions") + private List<DimensionDesc> dimensions; + @JsonProperty("measures") + private List<MeasureDesc> measures; + @JsonProperty("rowkey") + private RowKeyDesc rowkey; + @JsonProperty("hbase_mapping") + private HBaseMappingDesc hbaseMapping; + @JsonProperty("signature") + private String signature; + @JsonProperty("notify_list") + private List<String> notifyList; + @JsonProperty("status_need_notify") + private List<String> statusNeedNotify = Collections.emptyList(); + + @JsonProperty("partition_date_start") + private long partitionDateStart = 0L; + @JsonProperty("partition_date_end") + private long partitionDateEnd = 3153600000000l; + @JsonProperty("auto_merge_time_ranges") + private long[] autoMergeTimeRanges; + @JsonProperty("retention_range") + private long retentionRange = 0; + + @JsonProperty("engine_type") + private int engineType = IEngineAware.ID_MR_V1; + @JsonProperty("storage_type") + private int storageType = IStorageAware.ID_HBASE; + @JsonProperty("override_kylin_properties") + private LinkedHashMap<String, String> overrideKylinProps = new LinkedHashMap<String, String>(); + + private Map<String, Map<String, TblColRef>> columnMap = new HashMap<String, Map<String, TblColRef>>(); + private LinkedHashSet<TblColRef> allColumns = new LinkedHashSet<TblColRef>(); + private LinkedHashSet<TblColRef> dimensionColumns = new LinkedHashSet<TblColRef>(); + + private Map<TblColRef, DeriveInfo> derivedToHostMap = Maps.newHashMap(); + private Map<Array<TblColRef>, List<DeriveInfo>> hostToDerivedMap = Maps.newHashMap(); + + public boolean isEnableSharding() { + //in the future may extend to other storage that is shard-able + return storageType == IStorageAware.ID_SHARDED_HBASE; + } + + /** + * Error messages during resolving json metadata + */ + private List<String> errors = new ArrayList<String>(); + + /** + * @return all columns this cube can support, including derived + */ + public Set<TblColRef> listAllColumns() { + return allColumns; + } + + /** + * @return dimension columns including derived, BUT NOT measures + */ + public Set<TblColRef> listDimensionColumnsIncludingDerived() { + return dimensionColumns; + } + + /** + * @return dimension columns excluding derived and measures + */ + public List<TblColRef> listDimensionColumnsExcludingDerived() { + List<TblColRef> result = new ArrayList<TblColRef>(); + for (TblColRef col : dimensionColumns) { + if (isDerived(col) == false) + result.add(col); + } + return result; + } + + /** + * Find FunctionDesc by Full Expression. + * + * @return + */ + public FunctionDesc findFunctionOnCube(FunctionDesc manualFunc) { + for (MeasureDesc m : measures) { + if (m.getFunction().equals(manualFunc)) + return m.getFunction(); + } + return null; + } + + public TblColRef findColumnRef(String table, String column) { + Map<String, TblColRef> cols = columnMap.get(table); + if (cols == null) + return null; + else + return cols.get(column); + } + + public DimensionDesc findDimensionByColumn(TblColRef col) { + for (DimensionDesc dim : dimensions) { + if (ArrayUtils.contains(dim.getColumnRefs(), col)) + return dim; + } + return null; + } + + public DimensionDesc findDimensionByTable(String lookupTableName) { + lookupTableName = lookupTableName.toUpperCase(); + for (DimensionDesc dim : dimensions) + if (dim.getTable() != null && dim.getTable().equals(lookupTableName)) + return dim; + return null; + } + + public DimensionDesc findDimensionByName(String dimName) { + dimName = dimName.toUpperCase(); + for (DimensionDesc dim : dimensions) { + if (dimName.equals(dim.getName())) + return dim; + } + return null; + } + + /** + * Get all functions from each measure. + * + * @return + */ + public List<FunctionDesc> listAllFunctions() { + List<FunctionDesc> functions = new ArrayList<FunctionDesc>(); + for (MeasureDesc m : measures) { + functions.add(m.getFunction()); + } + return functions; + } + + public boolean isDerived(TblColRef col) { + return derivedToHostMap.containsKey(col); + } + + public DeriveInfo getHostInfo(TblColRef derived) { + return derivedToHostMap.get(derived); + } + + public Map<Array<TblColRef>, List<DeriveInfo>> getHostToDerivedInfo(List<TblColRef> rowCols, Collection<TblColRef> wantedCols) { + Map<Array<TblColRef>, List<DeriveInfo>> result = new HashMap<Array<TblColRef>, List<DeriveInfo>>(); + for (Entry<Array<TblColRef>, List<DeriveInfo>> entry : hostToDerivedMap.entrySet()) { + Array<TblColRef> hostCols = entry.getKey(); + boolean hostOnRow = rowCols.containsAll(Arrays.asList(hostCols.data)); + if (!hostOnRow) + continue; + + List<DeriveInfo> wantedInfo = new ArrayList<DeriveInfo>(); + for (DeriveInfo info : entry.getValue()) { + if (wantedCols == null || Collections.disjoint(wantedCols, Arrays.asList(info.columns)) == false) // has + // any + // wanted + // columns? + wantedInfo.add(info); + } + + if (wantedInfo.size() > 0) + result.put(hostCols, wantedInfo); + } + return result; + } + + public String getResourcePath() { + return concatResourcePath(name); + } + + public static String concatResourcePath(String descName) { + return ResourceStore.CUBE_DESC_RESOURCE_ROOT + "/" + descName + MetadataConstants.FILE_SURFIX; + } + + // ============================================================================ + + public HBaseMappingDesc getHBaseMapping() { + return hbaseMapping; + } + + public void setHBaseMapping(HBaseMappingDesc hbaseMapping) { + this.hbaseMapping = hbaseMapping; + } + + public KylinConfig getConfig() { + return config; + } + + public void setConfig(KylinConfig config) { + this.config = config; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getModelName() { + return modelName; + } + + public void setModelName(String modelName) { + this.modelName = modelName; + } + + public DataModelDesc getModel() { + return model; + } + + public void setModel(DataModelDesc model) { + this.model = model; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public String getFactTable() { + return model.getFactTable(); + } + + public TableDesc getFactTableDesc() { + return model.getFactTableDesc(); + } + + public List<TableDesc> getLookupTableDescs() { + return model.getLookupTableDescs(); + } + + public String[] getNullStrings() { + return nullStrings; + } + + public List<DimensionDesc> getDimensions() { + return dimensions; + } + + public void setDimensions(List<DimensionDesc> dimensions) { + this.dimensions = dimensions; + } + + public List<MeasureDesc> getMeasures() { + return measures; + } + + public void setMeasures(List<MeasureDesc> measures) { + this.measures = measures; + } + + public RowKeyDesc getRowkey() { + return rowkey; + } + + public void setRowkey(RowKeyDesc rowkey) { + this.rowkey = rowkey; + } + + public String getSignature() { + return signature; + } + + public void setSignature(String signature) { + this.signature = signature; + } + + public List<String> getNotifyList() { + return notifyList; + } + + public void setNotifyList(List<String> notifyList) { + this.notifyList = notifyList; + } + + public List<String> getStatusNeedNotify() { + return statusNeedNotify; + } + + public void setStatusNeedNotify(List<String> statusNeedNotify) { + this.statusNeedNotify = statusNeedNotify; + } + + @Override + public boolean equals(Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + + CubeDesc cubeDesc = (CubeDesc) o; + + if (!name.equals(cubeDesc.getName())) + return false; + if (!getFactTable().equals(cubeDesc.getFactTable())) + return false; + + return true; + } + + @Override + public int hashCode() { + int result = 0; + result = 31 * result + name.hashCode(); + result = 31 * result + getFactTable().hashCode(); + return result; + } + + @Override + public String toString() { + return "CubeDesc [name=" + name + "]"; + } + + public boolean checkSignature() { + if (StringUtils.isBlank(getSignature())) { + return true; + } + return calculateSignature().equals(getSignature()); + } + + public String calculateSignature() { + MessageDigest md = null; + try { + md = MessageDigest.getInstance("MD5"); + StringBuilder sigString = new StringBuilder(); + sigString.append(this.name).append("|").append(this.getFactTable()).append("|").append(JsonUtil.writeValueAsString(this.model.getPartitionDesc())).append("|").append(JsonUtil.writeValueAsString(this.dimensions)).append("|").append(JsonUtil.writeValueAsString(this.measures)).append("|").append(JsonUtil.writeValueAsString(this.rowkey)).append("|").append(JsonUtil.writeValueAsString(this.hbaseMapping)); + + byte[] signature = md.digest(sigString.toString().getBytes()); + return new String(Base64.encodeBase64(signature)); + } catch (NoSuchAlgorithmException | JsonProcessingException e) { + throw new RuntimeException("Failed to calculate signature"); + } + } + + public Map<String, TblColRef> buildColumnNameAbbreviation() { + Map<String, TblColRef> r = new CaseInsensitiveStringMap<TblColRef>(); + for (TblColRef col : listDimensionColumnsExcludingDerived()) { + r.put(col.getName(), col); + } + return r; + } + + public void init(KylinConfig config, Map<String, TableDesc> tables) { + this.errors.clear(); + this.config = config; + + if (this.modelName == null || this.modelName.length() == 0) { + this.addError("The cubeDesc '" + this.getName() + "' doesn't have data model specified."); + } + + this.model = MetadataManager.getInstance(config).getDataModelDesc(this.modelName); + + if (this.model == null) { + this.addError("No data model found with name '" + modelName + "'."); + } + + for (DimensionDesc dim : dimensions) { + dim.init(this, tables); + } + + sortDimAndMeasure(); + initDimensionColumns(); + initMeasureColumns(); + + rowkey.init(this); + if (hbaseMapping != null) { + hbaseMapping.init(this); + } + + initMeasureReferenceToColumnFamily(); + + // check all dimension columns are presented on rowkey + List<TblColRef> dimCols = listDimensionColumnsExcludingDerived(); + if (rowkey.getRowKeyColumns().length != dimCols.size()) { + addError("RowKey columns count (" + rowkey.getRowKeyColumns().length + ") does not match dimension columns count (" + dimCols.size() + "). "); + } + } + + private void initDimensionColumns() { + for (DimensionDesc dim : dimensions) { + JoinDesc join = dim.getJoin(); + + // init dimension columns + ArrayList<TblColRef> dimCols = Lists.newArrayList(); + String[] colStrs = dim.getColumn(); + + // when column is omitted, special case + if (colStrs == null && dim.isDerived() || ArrayUtils.contains(colStrs, "{FK}")) { + for (TblColRef col : join.getForeignKeyColumns()) { + dimCols.add(initDimensionColRef(col)); + } + } + // normal case + else { + if (colStrs == null || colStrs.length == 0) + throw new IllegalStateException("Dimension column must not be blank " + dim); + + for (String colStr : colStrs) { + dimCols.add(initDimensionColRef(dim, colStr)); + } + + // fill back column ref in hierarchy + if (dim.isHierarchy()) { + for (int i = 0; i < dimCols.size(); i++) + dim.getHierarchy()[i].setColumnRef(dimCols.get(i)); + } + } + + TblColRef[] dimColArray = (TblColRef[]) dimCols.toArray(new TblColRef[dimCols.size()]); + dim.setColumnRefs(dimColArray); + + // init derived columns + TblColRef[] hostCols = dimColArray; + if (dim.isDerived()) { + String[] derived = dim.getDerived(); + String[][] split = splitDerivedColumnAndExtra(derived); + String[] derivedNames = split[0]; + String[] derivedExtra = split[1]; + TblColRef[] derivedCols = new TblColRef[derivedNames.length]; + for (int i = 0; i < derivedNames.length; i++) { + derivedCols[i] = initDimensionColRef(dim, derivedNames[i]); + } + initDerivedMap(hostCols, DeriveType.LOOKUP, dim, derivedCols, derivedExtra); + } + + // PK-FK derive the other side + if (join != null) { + TblColRef[] fk = join.getForeignKeyColumns(); + TblColRef[] pk = join.getPrimaryKeyColumns(); + + allColumns.addAll(Arrays.asList(fk)); + allColumns.addAll(Arrays.asList(pk)); + for (int i = 0; i < fk.length; i++) { + int find = ArrayUtils.indexOf(hostCols, fk[i]); + if (find >= 0) { + TblColRef derivedCol = initDimensionColRef(pk[i]); + initDerivedMap(hostCols[find], DeriveType.PK_FK, dim, derivedCol); + } + } + /** disable this code as we don't need fk be derived from pk + for (int i = 0; i < pk.length; i++) { + int find = ArrayUtils.indexOf(hostCols, pk[i]); + if (find >= 0) { + TblColRef derivedCol = initDimensionColRef(fk[i]); + initDerivedMap(hostCols[find], DeriveType.PK_FK, dim, derivedCol); + } + } + */ + } + } + } + + private String[][] splitDerivedColumnAndExtra(String[] derived) { + String[] cols = new String[derived.length]; + String[] extra = new String[derived.length]; + for (int i = 0; i < derived.length; i++) { + String str = derived[i]; + int cut = str.indexOf(":"); + if (cut >= 0) { + cols[i] = str.substring(0, cut); + extra[i] = str.substring(cut + 1).trim(); + } else { + cols[i] = str; + extra[i] = ""; + } + } + return new String[][] { cols, extra }; + } + + private void initDerivedMap(TblColRef hostCol, DeriveType type, DimensionDesc dimension, TblColRef derivedCol) { + initDerivedMap(new TblColRef[] { hostCol }, type, dimension, new TblColRef[] { derivedCol }, null); + } + + private void initDerivedMap(TblColRef[] hostCols, DeriveType type, DimensionDesc dimension, TblColRef[] derivedCols, String[] extra) { + if (hostCols.length == 0 || derivedCols.length == 0) + throw new IllegalStateException("host/derived columns must not be empty"); + + // Although FK derives PK automatically, user unaware of this can declare PK as derived dimension explicitly. + // In that case, derivedCols[] will contain a FK which is transformed from the PK by initDimensionColRef(). + // Must drop FK from derivedCols[] before continue. + for (int i = 0; i < derivedCols.length; i++) { + if (ArrayUtils.contains(hostCols, derivedCols[i])) { + derivedCols = (TblColRef[]) ArrayUtils.remove(derivedCols, i); + extra = (String[]) ArrayUtils.remove(extra, i); + i--; + } + } + + Array<TblColRef> hostColArray = new Array<TblColRef>(hostCols); + List<DeriveInfo> infoList = hostToDerivedMap.get(hostColArray); + if (infoList == null) { + hostToDerivedMap.put(hostColArray, infoList = new ArrayList<DeriveInfo>()); + } + infoList.add(new DeriveInfo(type, dimension, derivedCols, false)); + + for (int i = 0; i < derivedCols.length; i++) { + TblColRef derivedCol = derivedCols[i]; + boolean isOneToOne = type == DeriveType.PK_FK || ArrayUtils.contains(hostCols, derivedCol) || (extra != null && extra[i].contains("1-1")); + derivedToHostMap.put(derivedCol, new DeriveInfo(type, dimension, hostCols, isOneToOne)); + } + } + + private TblColRef initDimensionColRef(DimensionDesc dim, String colName) { + TableDesc table = dim.getTableDesc(); + ColumnDesc col = table.findColumnByName(colName); + if (col == null) + throw new IllegalArgumentException("No column '" + colName + "' found in table " + table); + + TblColRef ref = new TblColRef(col); + + // always use FK instead PK, FK could be shared by more than one lookup tables + JoinDesc join = dim.getJoin(); + if (join != null) { + int idx = ArrayUtils.indexOf(join.getPrimaryKeyColumns(), ref); + if (idx >= 0) { + ref = join.getForeignKeyColumns()[idx]; + } + } + return initDimensionColRef(ref); + } + + private TblColRef initDimensionColRef(TblColRef ref) { + TblColRef existing = findColumnRef(ref.getTable(), ref.getName()); + if (existing != null) { + return existing; + } + + allColumns.add(ref); + dimensionColumns.add(ref); + + Map<String, TblColRef> cols = columnMap.get(ref.getTable()); + if (cols == null) { + columnMap.put(ref.getTable(), cols = new HashMap<String, TblColRef>()); + } + cols.put(ref.getName(), ref); + return ref; + } + + private void initMeasureColumns() { + if (measures == null || measures.isEmpty()) { + return; + } + + TableDesc factTable = getFactTableDesc(); + List<TableDesc> lookups = getLookupTableDescs(); + for (MeasureDesc m : measures) { + m.setName(m.getName().toUpperCase()); + + if (m.getDependentMeasureRef() != null) { + m.setDependentMeasureRef(m.getDependentMeasureRef().toUpperCase()); + } + + FunctionDesc func = m.getFunction(); + func.init(factTable, lookups); + allColumns.addAll(func.getParameter().getColRefs()); + +// // verify holistic count distinct as a dependent measure +// if (isHolisticCountDistinct() && StringUtils.isBlank(m.getDependentMeasureRef())) { +// throw new IllegalStateException(m + " is a holistic count distinct but it has no DependentMeasureRef defined!"); +// } + } + } + + + private void initMeasureReferenceToColumnFamily() { + if (measures == null || measures.size() == 0) + return; + + Map<String, MeasureDesc> measureLookup = new HashMap<String, MeasureDesc>(); + for (MeasureDesc m : measures) + measureLookup.put(m.getName(), m); + Map<String, Integer> measureIndexLookup = new HashMap<String, Integer>(); + for (int i = 0; i < measures.size(); i++) + measureIndexLookup.put(measures.get(i).getName(), i); + + for (HBaseColumnFamilyDesc cf : getHBaseMapping().getColumnFamily()) { + for (HBaseColumnDesc c : cf.getColumns()) { + String[] colMeasureRefs = c.getMeasureRefs(); + MeasureDesc[] measureDescs = new MeasureDesc[colMeasureRefs.length]; + int[] measureIndex = new int[colMeasureRefs.length]; + for (int i = 0; i < colMeasureRefs.length; i++) { + measureDescs[i] = measureLookup.get(colMeasureRefs[i]); + measureIndex[i] = measureIndexLookup.get(colMeasureRefs[i]); + } + c.setMeasures(measureDescs); + c.setMeasureIndex(measureIndex); + c.setColumnFamilyName(cf.getName()); + } + } + } + + private void sortDimAndMeasure() { + sortDimensionsByID(); + sortMeasuresByID(); + for (DimensionDesc dim : dimensions) { + sortHierarchiesByLevel(dim.getHierarchy()); + } + } + + private void sortDimensionsByID() { + Collections.sort(dimensions, new Comparator<DimensionDesc>() { + @Override + public int compare(DimensionDesc d1, DimensionDesc d2) { + Integer id1 = d1.getId(); + Integer id2 = d2.getId(); + return id1.compareTo(id2); + } + }); + } + + private void sortMeasuresByID() { + if (measures == null) { + measures = Lists.newArrayList(); + } + + // Collections.sort(measures, new Comparator<MeasureDesc>() { + // @Override + // public int compare(MeasureDesc m1, MeasureDesc m2) { + // Integer id1 = m1.getId(); + // Integer id2 = m2.getId(); + // return id1.compareTo(id2); + // } + // }); + } + + private void sortHierarchiesByLevel(HierarchyDesc[] hierarchies) { + if (hierarchies != null) { + Arrays.sort(hierarchies, new Comparator<HierarchyDesc>() { + @Override + public int compare(HierarchyDesc h1, HierarchyDesc h2) { + Integer level1 = Integer.parseInt(h1.getLevel()); + Integer level2 = Integer.parseInt(h2.getLevel()); + return level1.compareTo(level2); + } + }); + } + } + + public long getRetentionRange() { + return retentionRange; + } + + public void setRetentionRange(long retentionRange) { + this.retentionRange = retentionRange; + } + + public long[] getAutoMergeTimeRanges() { + return autoMergeTimeRanges; + } + + public void setAutoMergeTimeRanges(long[] autoMergeTimeRanges) { + this.autoMergeTimeRanges = autoMergeTimeRanges; + } + + /** + * Add error info and thrown exception out + * + * @param message + */ + public void addError(String message) { + addError(message, false); + } + + /** + * @param message error message + * @param silent if throw exception + */ + public void addError(String message, boolean silent) { + if (!silent) { + throw new IllegalStateException(message); + } else { + this.errors.add(message); + } + } + + public List<String> getError() { + return this.errors; + } + + public HBaseMappingDesc getHbaseMapping() { + return hbaseMapping; + } + + public void setHbaseMapping(HBaseMappingDesc hbaseMapping) { + this.hbaseMapping = hbaseMapping; + } + + public void setNullStrings(String[] nullStrings) { + this.nullStrings = nullStrings; + } + + public int getStorageType() { + return storageType; + } + + public void setStorageType(int storageType) { + this.storageType = storageType; + } + + public int getEngineType() { + return engineType; + } + + public void setEngineType(int engineType) { + this.engineType = engineType; + } + + public long getPartitionDateStart() { + return partitionDateStart; + } + + public void setPartitionDateStart(long partitionDateStart) { + this.partitionDateStart = partitionDateStart; + } + + public long getPartitionDateEnd() { + return partitionDateEnd; + } + + public void setPartitionDateEnd(long partitionDateEnd) { + this.partitionDateEnd = partitionDateEnd; + } + + public static CubeDesc getCopyOf(CubeDesc cubeDesc) { + CubeDesc newCubeDesc = new CubeDesc(); + newCubeDesc.setName(cubeDesc.getName()); + newCubeDesc.setModelName(cubeDesc.getModelName()); + newCubeDesc.setDescription(cubeDesc.getDescription()); + newCubeDesc.setNullStrings(cubeDesc.getNullStrings()); + newCubeDesc.setDimensions(cubeDesc.getDimensions()); + newCubeDesc.setMeasures(cubeDesc.getMeasures()); + newCubeDesc.setRowkey(cubeDesc.getRowkey()); + newCubeDesc.setHBaseMapping(cubeDesc.getHBaseMapping()); + newCubeDesc.setSignature(cubeDesc.getSignature()); + newCubeDesc.setNotifyList(cubeDesc.getNotifyList()); + newCubeDesc.setStatusNeedNotify(cubeDesc.getStatusNeedNotify()); + newCubeDesc.setAutoMergeTimeRanges(cubeDesc.getAutoMergeTimeRanges()); + newCubeDesc.setPartitionDateStart(cubeDesc.getPartitionDateStart()); + newCubeDesc.setPartitionDateEnd(cubeDesc.getPartitionDateEnd()); + newCubeDesc.setRetentionRange(cubeDesc.getRetentionRange()); + newCubeDesc.setEngineType(cubeDesc.getEngineType()); + newCubeDesc.setStorageType(cubeDesc.getStorageType()); + newCubeDesc.setConfig(cubeDesc.getConfig()); + newCubeDesc.updateRandomUuid(); + return newCubeDesc; + } +} http://git-wip-us.apache.org/repos/asf/kylin/blob/2512fb6f/core-cube/src/main/java/org/apache/kylin/cube/model/v1_4_0/DimensionDesc.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/v1_4_0/DimensionDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/v1_4_0/DimensionDesc.java new file mode 100644 index 0000000..f97f939 --- /dev/null +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/v1_4_0/DimensionDesc.java @@ -0,0 +1,239 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kylin.cube.model.v1_4_0; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +import org.apache.kylin.common.util.StringUtil; +import org.apache.kylin.metadata.model.JoinDesc; +import org.apache.kylin.metadata.model.LookupDesc; +import org.apache.kylin.metadata.model.TableDesc; +import org.apache.kylin.metadata.model.TblColRef; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + */ +@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE) +public class DimensionDesc { + + @JsonProperty("id") + private int id; + + @JsonProperty("name") + private String name; + + @JsonProperty("hierarchy") + private boolean isHierarchy; + @JsonProperty("table") + private String table; + @JsonProperty("column") + private String[] column; + @JsonProperty("derived") + private String[] derived; + + private TableDesc tableDesc; + private JoinDesc join; + private HierarchyDesc[] hierarchy; + + // computed + private TblColRef[] columnRefs; + private TblColRef[] derivedColRefs; + + public void init(CubeDesc cubeDesc, Map<String, TableDesc> tables) { + if (name != null) + name = name.toUpperCase(); + + if (table != null) + table = table.toUpperCase(); + + tableDesc = tables.get(this.getTable()); + if (tableDesc == null) + throw new IllegalStateException("Can't find table " + table + " for dimension " + name); + + join = null; + for (LookupDesc lookup : cubeDesc.getModel().getLookups()) { + if (lookup.getTable().equalsIgnoreCase(this.getTable())) { + join = lookup.getJoin(); + break; + } + } + + if (isHierarchy && this.column.length > 0) { + List<HierarchyDesc> hierarchyList = new ArrayList<HierarchyDesc>(3); + for (int i = 0, n = this.column.length; i < n; i++) { + String aColumn = this.column[i]; + HierarchyDesc aHierarchy = new HierarchyDesc(); + aHierarchy.setLevel(String.valueOf(i + 1)); + aHierarchy.setColumn(aColumn); + hierarchyList.add(aHierarchy); + } + + this.hierarchy = hierarchyList.toArray(new HierarchyDesc[hierarchyList.size()]); + } + + if (hierarchy != null && hierarchy.length == 0) + hierarchy = null; + if (derived != null && derived.length == 0) + derived = null; + + if (hierarchy != null) { + for (HierarchyDesc h : hierarchy) + h.setColumn(h.getColumn().toUpperCase()); + } + + if (derived != null) { + StringUtil.toUpperCaseArray(derived, derived); + } + + if (derived != null && join == null) { + throw new IllegalStateException("Derived can only be defined on lookup table, cube " + cubeDesc + ", " + this); + } + } + + public boolean isHierarchyColumn(TblColRef col) { + if (hierarchy == null) + return false; + + for (HierarchyDesc hier : hierarchy) { + if (hier.getColumnRef().equals(col)) + return true; + } + return false; + } + + public boolean isDerived() { + return derived != null; + } + + public boolean isHierarchy() { + return isHierarchy; + } + + public void setHierarchy(boolean isHierarchy) { + this.isHierarchy = isHierarchy; + } + + public String getTable() { + return table; + } + + public void setTable(String table) { + this.table = table; + } + + public int getId() { + return id; + } + + public void setId(int id) { + this.id = id; + } + + public JoinDesc getJoin() { + return join; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public TblColRef[] getColumnRefs() { + return this.columnRefs; + } + + public void setColumnRefs(TblColRef[] colRefs) { + this.columnRefs = colRefs; + } + + public String[] getColumn() { + return this.column; + } + + public void setColumn(String[] column) { + this.column = column; + } + + public HierarchyDesc[] getHierarchy() { + return hierarchy; + } + + public void setHierarchy(HierarchyDesc[] hierarchy) { + this.hierarchy = hierarchy; + } + + public String[] getDerived() { + return derived; + } + + public void setDerived(String[] derived) { + this.derived = derived; + } + + public TblColRef[] getDerivedColRefs() { + return derivedColRefs; + } + + public void setDerivedColRefs(TblColRef[] derivedColRefs) { + this.derivedColRefs = derivedColRefs; + } + + public TableDesc getTableDesc() { + return this.tableDesc; + } + + @Override + public boolean equals(Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + + DimensionDesc that = (DimensionDesc) o; + + if (id != that.id) + return false; + if (!name.equals(that.name)) + return false; + + return true; + } + + @Override + public int hashCode() { + int result = id; + result = 31 * result + name.hashCode(); + return result; + } + + @Override + public String toString() { + return "DimensionDesc [name=" + name + ", join=" + join + ", hierarchy=" + Arrays.toString(hierarchy) + ", table=" + table + ", column=" + Arrays.toString(column) + ", derived=" + Arrays.toString(derived) + "]"; + } + +} http://git-wip-us.apache.org/repos/asf/kylin/blob/2512fb6f/core-cube/src/main/java/org/apache/kylin/cube/model/v1_4_0/HBaseColumnDesc.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/v1_4_0/HBaseColumnDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/v1_4_0/HBaseColumnDesc.java new file mode 100644 index 0000000..2db4ec7 --- /dev/null +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/v1_4_0/HBaseColumnDesc.java @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kylin.cube.model.v1_4_0; + +import java.util.Arrays; + +import org.apache.kylin.metadata.model.FunctionDesc; +import org.apache.kylin.metadata.model.MeasureDesc; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + */ +@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE) +public class HBaseColumnDesc { + + @JsonProperty("qualifier") + private String qualifier; + @JsonProperty("measure_refs") + private String[] measureRefs; + + // these two will be assembled at runtime + private MeasureDesc[] measures; + private int[] measureIndex; // the index on CubeDesc.getMeasures() + private String columnFamilyName; + + public String getQualifier() { + return qualifier; + } + + public void setQualifier(String qualifier) { + this.qualifier = qualifier; + } + + public String[] getMeasureRefs() { + return measureRefs; + } + + public void setMeasureRefs(String[] measureRefs) { + this.measureRefs = measureRefs; + } + + public int[] getMeasureIndex() { + return measureIndex; + } + + public void setMeasureIndex(int[] index) { + this.measureIndex = index; + } + + public MeasureDesc[] getMeasures() { + return measures; + } + + public void setMeasures(MeasureDesc[] measures) { + this.measures = measures; + } + + public String getColumnFamilyName() { + return columnFamilyName; + } + + public void setColumnFamilyName(String columnFamilyName) { + this.columnFamilyName = columnFamilyName; + } + + public int findMeasure(FunctionDesc function) { + for (int i = 0; i < measures.length; i++) { + if (measures[i].getFunction().equals(function)) { + return i; + } + } + return -1; + } + + public boolean containsMeasure(String refName) { + for (String ref : measureRefs) { + if (ref.equals(refName)) + return true; + } + return false; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((columnFamilyName == null) ? 0 : columnFamilyName.hashCode()); + result = prime * result + ((qualifier == null) ? 0 : qualifier.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + HBaseColumnDesc other = (HBaseColumnDesc) obj; + if (columnFamilyName == null) { + if (other.columnFamilyName != null) + return false; + } else if (!columnFamilyName.equals(other.columnFamilyName)) + return false; + if (qualifier == null) { + if (other.qualifier != null) + return false; + } else if (!qualifier.equals(other.qualifier)) + return false; + return true; + } + + @Override + public String toString() { + return "HBaseColumnDesc [qualifier=" + qualifier + ", measureRefs=" + Arrays.toString(measureRefs) + "]"; + } + +} http://git-wip-us.apache.org/repos/asf/kylin/blob/2512fb6f/core-cube/src/main/java/org/apache/kylin/cube/model/v1_4_0/HBaseColumnFamilyDesc.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/v1_4_0/HBaseColumnFamilyDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/v1_4_0/HBaseColumnFamilyDesc.java new file mode 100644 index 0000000..bcde3d5 --- /dev/null +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/v1_4_0/HBaseColumnFamilyDesc.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kylin.cube.model.v1_4_0; + +import java.util.Arrays; + + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + */ +@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE) +public class HBaseColumnFamilyDesc { + + @JsonProperty("name") + private String name; + @JsonProperty("columns") + private HBaseColumnDesc[] columns; + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public HBaseColumnDesc[] getColumns() { + return columns; + } + + public void setColumns(HBaseColumnDesc[] columns) { + this.columns = columns; + } + + @Override + public String toString() { + return "HBaseColumnFamilyDesc [name=" + name + ", columns=" + Arrays.toString(columns) + "]"; + } + +} http://git-wip-us.apache.org/repos/asf/kylin/blob/2512fb6f/core-cube/src/main/java/org/apache/kylin/cube/model/v1_4_0/HBaseMappingDesc.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/v1_4_0/HBaseMappingDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/v1_4_0/HBaseMappingDesc.java new file mode 100644 index 0000000..95c9149 --- /dev/null +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/v1_4_0/HBaseMappingDesc.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kylin.cube.model.v1_4_0; + +import java.util.Arrays; +import java.util.Collection; +import java.util.LinkedList; + +import org.apache.kylin.common.util.StringUtil; +import org.apache.kylin.metadata.model.FunctionDesc; +import org.apache.kylin.metadata.model.MeasureDesc; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + */ +@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE) +public class HBaseMappingDesc { + + @JsonProperty("column_family") + private HBaseColumnFamilyDesc[] columnFamily; + + // point to the cube instance which contain this HBaseMappingDesc instance. + private CubeDesc cubeRef; + + public Collection<HBaseColumnDesc> findHBaseColumnByFunction(FunctionDesc function) { + Collection<HBaseColumnDesc> result = new LinkedList<HBaseColumnDesc>(); + HBaseMappingDesc hbaseMapping = cubeRef.getHBaseMapping(); + if (hbaseMapping == null || hbaseMapping.getColumnFamily() == null) { + return result; + } + for (HBaseColumnFamilyDesc cf : hbaseMapping.getColumnFamily()) { + for (HBaseColumnDesc c : cf.getColumns()) { + for (MeasureDesc m : c.getMeasures()) { + if (m.getFunction().equals(function)) { + result.add(c); + } + } + } + } + return result; + } + + public CubeDesc getCubeRef() { + return cubeRef; + } + + public void setCubeRef(CubeDesc cubeRef) { + this.cubeRef = cubeRef; + } + + public HBaseColumnFamilyDesc[] getColumnFamily() { + return columnFamily; + } + + public void setColumnFamily(HBaseColumnFamilyDesc[] columnFamily) { + this.columnFamily = columnFamily; + } + + public void init(CubeDesc cubeDesc) { + cubeRef = cubeDesc; + + for (HBaseColumnFamilyDesc cf : columnFamily) { + cf.setName(cf.getName().toUpperCase()); + + for (HBaseColumnDesc c : cf.getColumns()) { + c.setQualifier(c.getQualifier().toUpperCase()); + StringUtil.toUpperCaseArray(c.getMeasureRefs(), c.getMeasureRefs()); + } + } + } + + @Override + public String toString() { + return "HBaseMappingDesc [columnFamily=" + Arrays.toString(columnFamily) + "]"; + } + +} http://git-wip-us.apache.org/repos/asf/kylin/blob/2512fb6f/core-cube/src/main/java/org/apache/kylin/cube/model/v1_4_0/HierarchyDesc.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/v1_4_0/HierarchyDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/v1_4_0/HierarchyDesc.java new file mode 100644 index 0000000..a679fdb --- /dev/null +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/v1_4_0/HierarchyDesc.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kylin.cube.model.v1_4_0; + +import org.apache.kylin.metadata.model.TblColRef; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + */ +@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE) +public class HierarchyDesc { + + @JsonProperty("level") + private String level; + @JsonProperty("column") + private String column; + + private TblColRef columnRef; + + public String getLevel() { + return level; + } + + public void setLevel(String level) { + this.level = level; + } + + public TblColRef getColumnRef() { + return columnRef; + } + + public void setColumnRef(TblColRef column) { + this.columnRef = column; + } + + public String getColumn() { + return column; + } + + public void setColumn(String columnName) { + this.column = columnName; + } + + @Override + public String toString() { + return "HierarchyDesc [level=" + level + ", column=" + column + "]"; + } + +} http://git-wip-us.apache.org/repos/asf/kylin/blob/2512fb6f/core-cube/src/main/java/org/apache/kylin/cube/model/v1_4_0/RowKeyColDesc.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/v1_4_0/RowKeyColDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/v1_4_0/RowKeyColDesc.java new file mode 100644 index 0000000..20245f4 --- /dev/null +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/v1_4_0/RowKeyColDesc.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kylin.cube.model.v1_4_0; + +import org.apache.kylin.metadata.model.TblColRef; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * @author yangli9 + * + */ +@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE) +public class RowKeyColDesc { + + @JsonProperty("column") + private String column; + @JsonProperty("length") + private int length; + @JsonProperty("dictionary") + private String dictionary; + @JsonProperty("mandatory") + private boolean mandatory = false; + + // computed + private int bitIndex; + private TblColRef colRef; + + public String getDictionary() { + return dictionary; + } + + public String getColumn() { + return column; + } + + void setColumn(String column) { + this.column = column; + } + + public int getLength() { + return length; + } + + public boolean isMandatory() { + return mandatory; + } + + public int getBitIndex() { + return bitIndex; + } + + void setBitIndex(int index) { + this.bitIndex = index; + } + + public TblColRef getColRef() { + return colRef; + } + + void setColRef(TblColRef colRef) { + this.colRef = colRef; + } + + public void setDictionary(String dictionary) { + this.dictionary = dictionary; + } + + @Override + public String toString() { + return "RowKeyColDesc [column=" + column + ", length=" + length + ", dictionary=" + dictionary + ", mandatory=" + mandatory + "]"; + } + +} http://git-wip-us.apache.org/repos/asf/kylin/blob/2512fb6f/core-cube/src/main/java/org/apache/kylin/cube/model/v1_4_0/RowKeyDesc.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/v1_4_0/RowKeyDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/v1_4_0/RowKeyDesc.java new file mode 100644 index 0000000..112a249 --- /dev/null +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/v1_4_0/RowKeyDesc.java @@ -0,0 +1,301 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kylin.cube.model.v1_4_0; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.commons.lang.StringUtils; +import org.apache.kylin.common.util.StringUtil; +import org.apache.kylin.metadata.model.TblColRef; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + */ +@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE) +public class RowKeyDesc { + + public static class HierarchyMask { + public long fullMask; + public long[] allMasks; + } + + public static class AggrGroupMask { + public AggrGroupMask(int size) { + groupOneBitMasks = new long[size]; + } + + public long groupMask; + public long groupOneBitMasks[]; + public long uniqueMask; + public long leftoverMask; + } + + @JsonProperty("rowkey_columns") + private RowKeyColDesc[] rowkeyColumns; + @JsonProperty("aggregation_groups") + private String[][] aggregationGroups; + + // computed content + private CubeDesc cubeDesc; + private Map<TblColRef, RowKeyColDesc> columnMap; + + private long fullMask; + private long mandatoryColumnMask; + private AggrGroupMask[] aggrGroupMasks; + private long aggrGroupFullMask; + private long hierarchyFullMask; + private long tailMask; + + private List<HierarchyMask> hierarchyMasks; + + public RowKeyColDesc[] getRowKeyColumns() { + return rowkeyColumns; + } + + // search a specific row key col + public int getRowKeyIndexByColumnName(String columnName) { + if (this.rowkeyColumns == null) + return -1; + + for (int i = 0; i < this.rowkeyColumns.length; ++i) { + RowKeyColDesc desc = this.rowkeyColumns[i]; + if (desc.getColumn().equalsIgnoreCase(columnName)) { + return i; + } + } + return -1; + } + + public int getNCuboidBuildLevels() { + // N aggregation columns requires N levels of cuboid build + // - N columns requires N-1 levels build + // - zero tail cuboid needs one more additional level + Set<String> aggDims = new HashSet<String>(); + for (String[] aggrGroup : aggregationGroups) { + for (String dim : aggrGroup) { + aggDims.add(dim); + } + } + return aggDims.size(); + } + + public String[][] getAggregationGroups() { + return aggregationGroups; + } + + public CubeDesc getCubeRef() { + return cubeDesc; + } + + public void setCubeRef(CubeDesc cubeRef) { + this.cubeDesc = cubeRef; + } + + public long getFullMask() { + return fullMask; + } + + public long getMandatoryColumnMask() { + return mandatoryColumnMask; + } + + public long getAggrGroupFullMask() { + return aggrGroupFullMask; + } + + public AggrGroupMask[] getAggrGroupMasks() { + return aggrGroupMasks; + } + + public List<HierarchyMask> getHierarchyMasks() { + return hierarchyMasks; + } + + public long getHierarchyFullMask() { + return hierarchyFullMask; + } + + public long getTailMask() { + return tailMask; + } + + public int getColumnBitIndex(TblColRef col) { + return getColDesc(col).getBitIndex(); + } + + public int getColumnLength(TblColRef col) { + return getColDesc(col).getLength(); + } + + public String getDictionary(TblColRef col) { + return getColDesc(col).getDictionary(); + } + + private RowKeyColDesc getColDesc(TblColRef col) { + RowKeyColDesc desc = columnMap.get(col); + if (desc == null) + throw new NullPointerException("Column " + col + " does not exist in row key desc"); + return desc; + } + + public boolean isUseDictionary(int index) { + String useDictionary = rowkeyColumns[index].getDictionary(); + return useDictionary(useDictionary); + } + + public boolean isUseDictionary(TblColRef col) { + String useDictionary = getDictionary(col); + return useDictionary(useDictionary); + } + + private boolean useDictionary(String useDictionary) { + return !StringUtils.isBlank(useDictionary) && !"false".equals(useDictionary); + } + + public void init(CubeDesc cube) { + setCubeRef(cube); + Map<String, TblColRef> colNameAbbr = cube.buildColumnNameAbbreviation(); + + buildRowKey(colNameAbbr); + buildAggregationGroups(colNameAbbr); + buildHierarchyMasks(); + } + + @Override + public String toString() { + return "RowKeyDesc [rowkeyColumns=" + Arrays.toString(rowkeyColumns) + ", aggregationGroups=" + Arrays.toString(aggregationGroups) + "]"; + } + + private void buildRowKey(Map<String, TblColRef> colNameAbbr) { + columnMap = new HashMap<TblColRef, RowKeyColDesc>(); + mandatoryColumnMask = 0; + + for (int i = 0; i < rowkeyColumns.length; i++) { + RowKeyColDesc rowKeyColDesc = rowkeyColumns[i]; + String column = rowKeyColDesc.getColumn(); + rowKeyColDesc.setColumn(column.toUpperCase()); + rowKeyColDesc.setBitIndex(rowkeyColumns.length - i - 1); + rowKeyColDesc.setColRef(colNameAbbr.get(column)); + if (rowKeyColDesc.getColRef() == null) { + throw new IllegalArgumentException("Cannot find rowkey column " + column + " in cube " + cubeDesc); + } + + columnMap.put(rowKeyColDesc.getColRef(), rowKeyColDesc); + + if (rowKeyColDesc.isMandatory()) { + mandatoryColumnMask |= 1L << rowKeyColDesc.getBitIndex(); + } + } + } + + private void buildAggregationGroups(Map<String, TblColRef> colNameAbbr) { + if (aggregationGroups == null) { + aggregationGroups = new String[0][]; + } + + for (int i = 0; i < aggregationGroups.length; i++) { + StringUtil.toUpperCaseArray(aggregationGroups[i], this.aggregationGroups[i]); + } + + for (int i = 0; i < this.rowkeyColumns.length; i++) { + int index = rowkeyColumns[i].getBitIndex(); + this.fullMask |= 1L << index; + } + + this.aggrGroupMasks = new AggrGroupMask[aggregationGroups.length]; + for (int i = 0; i < this.aggregationGroups.length; i++) { + String[] aggGrp = this.aggregationGroups[i]; + AggrGroupMask mask = new AggrGroupMask(aggGrp.length); + + for (int j = 0; j < aggGrp.length; j++) { + TblColRef aggCol = colNameAbbr.get(aggGrp[j].toUpperCase()); + if (aggCol == null) { + throw new IllegalArgumentException("Can't find aggregation column " + aggGrp[j] + " in cube " + this.cubeDesc.getName()); + } + Integer index = getColumnBitIndex(aggCol); + mask.groupMask |= 1L << index; + mask.groupOneBitMasks[j] = 1L << index; + this.aggrGroupFullMask |= 1L << index; + } + this.aggrGroupMasks[i] = mask; + } + + this.tailMask = fullMask ^ mandatoryColumnMask ^ aggrGroupFullMask; + + // unique mask = (bits in this group) - (bits in following groups) + // leftover mask = (tail bits) + (bits in following groups) - (bits in + // this group) + for (int i = 0; i < aggrGroupMasks.length; i++) { + AggrGroupMask mask = aggrGroupMasks[i]; + + mask.uniqueMask = mask.groupMask; + for (int j = i + 1; j < aggrGroupMasks.length; j++) { + mask.uniqueMask &= ~aggrGroupMasks[j].groupMask; + } + + mask.leftoverMask = tailMask; + for (int j = i + 1; j < aggrGroupMasks.length; j++) { + mask.leftoverMask |= aggrGroupMasks[j].groupMask; + } + mask.leftoverMask &= ~mask.groupMask; + } + } + + private void buildHierarchyMasks() { + this.hierarchyMasks = new ArrayList<HierarchyMask>(); + + for (DimensionDesc dimension : this.cubeDesc.getDimensions()) { + HierarchyDesc[] hierarchies = dimension.getHierarchy(); + if (hierarchies == null || hierarchies.length == 0) + continue; + + HierarchyMask mask = new HierarchyMask(); + ArrayList<Long> allMaskList = new ArrayList<Long>(); + for (int i = 0; i < hierarchies.length; i++) { + TblColRef hColumn = hierarchies[i].getColumnRef(); + Integer index = getColumnBitIndex(hColumn); + long bit = 1L << index; + + if ((tailMask & bit) > 0) + continue; // ignore levels in tail, they don't participate + // aggregation group combination anyway + + mask.fullMask |= bit; + this.hierarchyFullMask |= bit; + allMaskList.add(mask.fullMask); + } + + mask.allMasks = new long[allMaskList.size()]; + for (int i = 0; i < allMaskList.size(); i++) + mask.allMasks[i] = allMaskList.get(i); + + this.hierarchyMasks.add(mask); + } + } + +} http://git-wip-us.apache.org/repos/asf/kylin/blob/2512fb6f/core-cube/src/main/java/org/apache/kylin/cube/model/v3/CubeBuildTypeEnum.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/v3/CubeBuildTypeEnum.java b/core-cube/src/main/java/org/apache/kylin/cube/model/v3/CubeBuildTypeEnum.java deleted file mode 100644 index 914dac2..0000000 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/v3/CubeBuildTypeEnum.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.cube.model.v3; - -/** - * @author xduo - * - */ -public enum CubeBuildTypeEnum { - /** - * rebuild a segment or incremental build - */ - BUILD, - /** - * merge segments - */ - MERGE, - - /** - * refresh segments - */ - REFRESH -}
