Repository: kylin Updated Branches: refs/heads/master f1ecd8ef9 -> 24b7d9105
http://git-wip-us.apache.org/repos/asf/kylin/blob/2512fb6f/core-cube/src/main/java/org/apache/kylin/cube/upgrade/V1_5_0/CubeMetadataUpgrade_v_1_5_0.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/upgrade/V1_5_0/CubeMetadataUpgrade_v_1_5_0.java b/core-cube/src/main/java/org/apache/kylin/cube/upgrade/V1_5_0/CubeMetadataUpgrade_v_1_5_0.java new file mode 100644 index 0000000..878379f --- /dev/null +++ b/core-cube/src/main/java/org/apache/kylin/cube/upgrade/V1_5_0/CubeMetadataUpgrade_v_1_5_0.java @@ -0,0 +1,150 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kylin.cube.upgrade.V1_5_0; + +import java.io.IOException; +import java.util.List; + +import org.apache.kylin.common.persistence.ResourceStore; +import org.apache.kylin.cube.CubeDescManager; +import org.apache.kylin.cube.CubeInstance; +import org.apache.kylin.cube.CubeManager; +import org.apache.kylin.cube.model.CubeDesc; +import org.apache.kylin.cube.upgrade.common.CubeMetadataUpgrade; +import org.apache.kylin.cube.upgrade.common.MetadataVersionRefresher; +import org.apache.kylin.metadata.model.IEngineAware; +import org.apache.kylin.metadata.model.IStorageAware; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * In 1.0, 1.1, 1.2 and 1.3 release there was a CubeMetadataUpgrade which is actually CubeMetadataUpgradeV1, + * that upgrades metadata store from kylin-0.7 compatible to 1.0 ~ 1.3 compatible. The major difference is that we split + * cube desc to cube desc + model desc + * + * Metadata are backward compatible from 1.0 ~ 1.3. + * + * In 1.4.x there is a CubeMetadataUpgradeV2 which is responsible for upgrading the metadata store + * from 1.0 ~ 1.3 compatible to 1.4.x compatible. The major actions in that is revising cube desc signature and upgrade model desc. + * Since the Apache Kylin community never officially make a 1.4.x release. Normal users can ignore it. + * + * This CubeMetadataUpgrade_v_1_5 upgrades metadata store from 1.0 ~ 1.3 compatible to 1.5 compatible. + * the major differences are: + * 1) changes from 1.0 ~ 1.3 compatible to 1.4.x compatible + * 2) brand new definition of partial cubes to allow users to select cuboids more flexibly. https://issues.apache.org/jira/browse/KYLIN-242 + * + * Notice: + * 1) From Kylin 1.5.0, every metadata store entity will bind with release version number. See RootPersistentEntity.version + * 2) From Kylin 1.5.0, the CubeMetadataUpgrade class will be named after version number. As with CubeMetadataUpgrade_v_1_5_0 + * 3) For details on how to upgrade from prior 1.5 to 1.5 compatible please visit http://kylin.apache.org/docs15/howto/howto_upgrade.html. + */ +public class CubeMetadataUpgrade_v_1_5_0 extends CubeMetadataUpgrade { + private static final Logger logger = LoggerFactory.getLogger(CubeMetadataUpgrade_v_1_5_0.class); + + public CubeMetadataUpgrade_v_1_5_0(String newMetadataUrl) { + super(newMetadataUrl); + } + + public void upgradeNonCompatibleMeta() { + upgradeCubeDesc(); + } + + public void upgradeCompatibleMeta() { + upgradeVersion(); + clear(); + upgradeEngineTypeStorageType(); + upgradeSignature(); + } + + private void upgradeVersion() { + MetadataVersionRefresher refresher = new MetadataVersionRefresher(this.store); + try { + refresher.refresh(); + } catch (IOException e) { + throw new RuntimeException("Failed to upgrade version number", e); + } + } + + private void upgradeCubeDesc() { + List<String> paths = listResourceStore(ResourceStore.CUBE_DESC_RESOURCE_ROOT); + for (String path : paths) { + logger.info("CubeMetadataUpgrade_v_1_5_0 handling in upgradeCubeDesc {}", path); + + try { + CubeDescUpgrade_v_1_5_0 upgrade = new CubeDescUpgrade_v_1_5_0(path, store); + CubeDesc ndesc = upgrade.upgrade(); + + ResourceStore.getStore(config).putResource(ndesc.getResourcePath(), ndesc, CubeDescManager.CUBE_DESC_SERIALIZER); + updatedResources.add(ndesc.getResourcePath()); + } catch (Exception e) { + logger.error("error", e); + errorMsgs.add("upgradeCubeDesc at '" + path + "' failed: " + e.getLocalizedMessage()); + } + } + } + + private void upgradeSignature() { + CubeDescManager cubeDescManager = CubeDescManager.getInstance(config); + List<CubeDesc> cubeDescs = cubeDescManager.listAllDesc(); + for (CubeDesc cubeDesc : cubeDescs) { + logger.info("CubeMetadataUpgrade_v_1_5_0 handling in upgradeSignature {}", cubeDesc.getName()); + upgradeSignature(cubeDesc); + } + } + + private void upgradeSignature(CubeDesc cubeDesc) { + try { + String calculatedSign = cubeDesc.calculateSignature(); + if (cubeDesc.getSignature() == null || (!cubeDesc.getSignature().equals(calculatedSign))) { + cubeDesc.setSignature(calculatedSign); + store.putResource(cubeDesc.getResourcePath(), cubeDesc, CubeDescManager.CUBE_DESC_SERIALIZER); + updatedResources.add(cubeDesc.getResourcePath()); + } + } catch (Exception e) { + logger.error("error", e); + errorMsgs.add("upgradeSignature [" + cubeDesc.getName() + "] failed: " + e.getLocalizedMessage()); + } + } + + // Update engine_type and storage_type to v2, if the cube has no segments. + private void upgradeEngineTypeStorageType() { + CubeManager cubeManager = CubeManager.getInstance(config); + List<CubeInstance> cubes = cubeManager.listAllCubes(); + for (CubeInstance cube : cubes) { + try { + org.apache.kylin.cube.model.CubeDesc cubeDesc = cube.getDescriptor(); + if (cube.getFirstSegment() == null && cubeDesc != null && cubeDesc.getStorageType() == IStorageAware.ID_HBASE && cubeDesc.getEngineType() == IEngineAware.ID_MR_V1) { + logger.info("CubeMetadataUpgrade_v_1_5_0 handling in upgradeEngineTypeStorageType {}", cube.getName()); + + cubeDesc.setEngineType(IEngineAware.ID_MR_V2); + cubeDesc.setStorageType(IStorageAware.ID_SHARDED_HBASE); + + store.putResource(cubeDesc.getResourcePath(), cubeDesc, CubeDescManager.CUBE_DESC_SERIALIZER); + updatedResources.add(cubeDesc.getResourcePath()); + } else { + logger.info("CubeDesc {}'s storage type and engine type will not be upgraded because they're not empty", cubeDesc.getName()); + } + } catch (Exception e) { + logger.error("error", e); + errorMsgs.add("upgradeEngineTypeStorageType [" + cube.getName() + "] failed: " + e.getLocalizedMessage()); + } + } + } + +} http://git-wip-us.apache.org/repos/asf/kylin/blob/2512fb6f/core-cube/src/main/java/org/apache/kylin/cube/upgrade/V3/CubeDescUpgraderV3.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/upgrade/V3/CubeDescUpgraderV3.java b/core-cube/src/main/java/org/apache/kylin/cube/upgrade/V3/CubeDescUpgraderV3.java deleted file mode 100644 index cc63671..0000000 --- a/core-cube/src/main/java/org/apache/kylin/cube/upgrade/V3/CubeDescUpgraderV3.java +++ /dev/null @@ -1,287 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.cube.upgrade.V3; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.util.Arrays; -import java.util.List; -import java.util.Set; -import java.util.TreeSet; - -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.kylin.common.KylinConfig; -import org.apache.kylin.common.persistence.JsonSerializer; -import org.apache.kylin.common.persistence.ResourceStore; -import org.apache.kylin.common.persistence.Serializer; -import org.apache.kylin.common.util.JsonUtil; -import org.apache.kylin.cube.model.AggregationGroup; -import org.apache.kylin.cube.model.SelectRule; -import org.apache.kylin.cube.model.v3.CubeDesc; -import org.apache.kylin.cube.model.v3.DimensionDesc; -import org.apache.kylin.cube.model.v3.HBaseMappingDesc; -import org.apache.kylin.cube.model.v3.RowKeyColDesc; -import org.apache.kylin.cube.model.v3.RowKeyDesc; - -import com.google.common.collect.Lists; - -public class CubeDescUpgraderV3 { - - @SuppressWarnings("unused") - private static final Log logger = LogFactory.getLog(CubeDescUpgraderV3.class); - private static final Serializer<CubeDesc> oldCubeDescSerializer = new JsonSerializer<CubeDesc>(CubeDesc.class); - - private String resourcePath; - - private List<String[]> oldHierarchies = Lists.newArrayList(); - private List<String> oldMandatories = Lists.newArrayList(); - private String[][] oldAggGroup = null; - private Set<String> allRowKeyCols = newIgnoreCaseSet(null); - - public CubeDescUpgraderV3(String resourcePath) { - this.resourcePath = resourcePath; - } - - public org.apache.kylin.cube.model.CubeDesc upgrade() throws IOException { - CubeDesc oldModel = loadOldCubeDesc(resourcePath); - - org.apache.kylin.cube.model.CubeDesc newModel = new org.apache.kylin.cube.model.CubeDesc(); - copyUnChangedProperties(oldModel, newModel); - upgradeDimension(oldModel, newModel); - upgradeRowKey(oldModel, newModel); - upgradeHBaseMapping(oldModel, newModel); - upgradeAggregationGroup(newModel);//must do at last - - return newModel; - } - - private CubeDesc loadOldCubeDesc(String path) throws IOException { - ResourceStore store = getStore(); - - CubeDesc ndesc = store.getResource(path, CubeDesc.class, oldCubeDescSerializer); - - if (StringUtils.isBlank(ndesc.getName())) { - throw new IllegalStateException("CubeDesc name must not be blank"); - } - - return ndesc; - } - - private Set<String> newIgnoreCaseSet(Set<String> input) { - Set<String> ret = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); - if (input != null) - ret.addAll(input); - return ret; - } - - private String[] toArray(Set<String> input) { - return input.toArray(new String[input.size()]); - } - - private boolean rowKeyColExistsInMultipleAggGroup() { - if (oldAggGroup == null) - return false; - - int total = 0; - Set<String> overall = newIgnoreCaseSet(null); - for (String[] group : oldAggGroup) { - Set<String> temp = newIgnoreCaseSet(null); - for (String entry : group) { - - overall.add(entry); - temp.add(entry); - } - total += temp.size(); - } - return overall.size() != total; - } - - private void upgradeAggregationGroup(org.apache.kylin.cube.model.CubeDesc newModel) { - - List<AggregationGroup> aggs = Lists.newArrayList(); - if (oldAggGroup == null || oldAggGroup.length == 0) { - oldAggGroup = new String[1][]; - oldAggGroup[0] = toArray(allRowKeyCols); - } - - if (rowKeyColExistsInMultipleAggGroup()) { - throw new IllegalArgumentException("rowKeyColExistsInMultipleAggGroup!"); - } - - Set<String> visited = newIgnoreCaseSet(null); - - for (String[] group : oldAggGroup) { - AggregationGroup agg = new AggregationGroup(); - - Set<String> remaining = newIgnoreCaseSet(allRowKeyCols); - remaining.removeAll(visited); - - Set<String> joint = newIgnoreCaseSet(remaining); - joint.removeAll(oldMandatories); - - Set<String> groupAsSet = newIgnoreCaseSet(null); - for (String entry : group) { - groupAsSet.add(entry); - } - visited.addAll(groupAsSet); - joint.removeAll(groupAsSet); - - List<String> mandatories = Lists.newArrayList(); - List<String[]> hierarchies = Lists.newArrayList(); - - for (String s : oldMandatories) { - mandatories.add(s); - } - - for (String[] h : oldHierarchies) { - if (groupAsSet.containsAll(Arrays.asList(h))) { - hierarchies.add(h); - } - } - - agg.setIncludes(toArray(remaining)); - - SelectRule selectRule = new SelectRule(); - selectRule.hierarchy_dims = hierarchies.toArray(new String[hierarchies.size()][]); - if (joint.size() != 0) { - selectRule.joint_dims = new String[1][]; - selectRule.joint_dims[0] = joint.toArray(new String[joint.size()]); - } else { - selectRule.joint_dims = new String[0][]; - } - selectRule.mandatory_dims = mandatories.toArray(new String[mandatories.size()]); - agg.setSelectRule(selectRule); - - aggs.add(agg); - - } - newModel.setAggregationGroups(aggs); - } - - private void upgradeDimension(CubeDesc oldModel, org.apache.kylin.cube.model.CubeDesc newModel) { - List<DimensionDesc> oldDimensions = oldModel.getDimensions(); - if (oldDimensions == null) { - throw new IllegalArgumentException("dimensions is null"); - } - List<org.apache.kylin.cube.model.DimensionDesc> newDimensions = Lists.newArrayList(); - - for (DimensionDesc oldDim : oldDimensions) { - if (oldDim.isDerived()) { - org.apache.kylin.cube.model.DimensionDesc newDim = new org.apache.kylin.cube.model.DimensionDesc(); - - newDim.setName(oldDim.getName()); - newDim.setTable(oldDim.getTable()); - newDim.setColumn("{FK}"); - newDim.setDerived(oldDim.getDerived()); - - newDimensions.add(newDim); - } else { - if (oldDim.isHierarchy()) { - oldHierarchies.add(oldDim.getColumn()); - } - - for (String columnStr : oldDim.getColumn()) { - org.apache.kylin.cube.model.DimensionDesc newDim = new org.apache.kylin.cube.model.DimensionDesc(); - - newDim.setName(oldDim.getName()); - newDim.setTable(oldDim.getTable()); - newDim.setColumn(columnStr); - newDim.setDerived(null); - - newDimensions.add(newDim); - } - } - } - - newModel.setDimensions(newDimensions); - } - - private void upgradeRowKey(CubeDesc oldModel, org.apache.kylin.cube.model.CubeDesc newModel) { - RowKeyDesc oldRowKey = oldModel.getRowkey(); - if (oldModel == null) { - throw new IllegalArgumentException("RowKeyDesc is null"); - } - - if (oldRowKey.getRowKeyColumns() == null) { - throw new IllegalArgumentException("RowKeyDesc.getRowKeyColumns is null"); - } - - org.apache.kylin.cube.model.RowKeyDesc newRowKey = new org.apache.kylin.cube.model.RowKeyDesc(); - org.apache.kylin.cube.model.RowKeyColDesc[] cols = new org.apache.kylin.cube.model.RowKeyColDesc[oldRowKey.getRowKeyColumns().length]; - int index = 0; - for (RowKeyColDesc oldRowKeyCol : oldRowKey.getRowKeyColumns()) { - org.apache.kylin.cube.model.RowKeyColDesc newRowKeyCol = new org.apache.kylin.cube.model.RowKeyColDesc(); - - allRowKeyCols.add(oldRowKeyCol.getColumn()); - if (oldRowKeyCol.isMandatory()) { - oldMandatories.add(oldRowKeyCol.getColumn()); - } - - newRowKeyCol.setColumn(oldRowKeyCol.getColumn()); - if (oldRowKeyCol.getDictionary() != null && "true".equalsIgnoreCase(oldRowKeyCol.getDictionary())) { - newRowKeyCol.setEncoding("dict"); - } else if (oldRowKeyCol.getLength() > 0) { - newRowKeyCol.setEncoding("fixed_length:" + oldRowKeyCol.getLength()); - } else { - throw new IllegalArgumentException("Unknow encoding: Dictionary " + oldRowKeyCol.getDictionary() + ", length: " + oldRowKeyCol.getLength()); - } - cols[index++] = newRowKeyCol; - } - oldAggGroup = oldRowKey.getAggregationGroups(); - - newRowKey.setRowkeyColumns(cols); - newModel.setRowkey(newRowKey); - } - - private void upgradeHBaseMapping(CubeDesc oldModel, org.apache.kylin.cube.model.CubeDesc newModel) { - HBaseMappingDesc hbaseMappingDesc = oldModel.getHBaseMapping(); - try { - - ByteArrayOutputStream os = new ByteArrayOutputStream(); - JsonUtil.writeValueIndent(os, hbaseMappingDesc); - byte[] blob = os.toByteArray(); - ByteArrayInputStream is = new ByteArrayInputStream(blob); - org.apache.kylin.cube.model.HBaseMappingDesc newHBaseMappingDesc = JsonUtil.readValue(is, org.apache.kylin.cube.model.HBaseMappingDesc.class); - newModel.setHbaseMapping(newHBaseMappingDesc); - - } catch (IOException e) { - throw new RuntimeException("error when copying HBaseMappingDesc"); - } - } - - private void copyUnChangedProperties(CubeDesc oldModel, org.apache.kylin.cube.model.CubeDesc newModel) { - newModel.setUuid(oldModel.getUuid()); - newModel.setName(oldModel.getName()); - newModel.setDescription(oldModel.getDescription()); - newModel.setMeasures(oldModel.getMeasures()); - newModel.setNullStrings(oldModel.getNullStrings()); - newModel.setModelName(oldModel.getModelName()); - newModel.setNotifyList(oldModel.getNotifyList()); - newModel.setLastModified(oldModel.getLastModified()); - newModel.setStorageType(oldModel.getStorageType()); - newModel.setEngineType(oldModel.getEngineType()); - } - - protected static ResourceStore getStore() { - return ResourceStore.getStore(KylinConfig.getInstanceFromEnv()); - } -} http://git-wip-us.apache.org/repos/asf/kylin/blob/2512fb6f/core-cube/src/main/java/org/apache/kylin/cube/upgrade/V3/CubeMetadataUpgradeV3.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/upgrade/V3/CubeMetadataUpgradeV3.java b/core-cube/src/main/java/org/apache/kylin/cube/upgrade/V3/CubeMetadataUpgradeV3.java deleted file mode 100644 index 4a36799..0000000 --- a/core-cube/src/main/java/org/apache/kylin/cube/upgrade/V3/CubeMetadataUpgradeV3.java +++ /dev/null @@ -1,180 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.cube.upgrade.V3; - -import java.io.File; -import java.io.IOException; -import java.util.List; - -import org.apache.commons.io.FileUtils; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.kylin.common.KylinConfig; -import org.apache.kylin.common.persistence.ResourceStore; -import org.apache.kylin.cube.CubeDescManager; -import org.apache.kylin.cube.CubeManager; -import org.apache.kylin.cube.model.CubeDesc; -import org.apache.kylin.metadata.MetadataConstants; -import org.apache.kylin.metadata.MetadataManager; -import org.apache.kylin.metadata.project.ProjectManager; - -import com.google.common.collect.Lists; - -/** - * back in 1.x branch there was a CubeMetadataUpgrade which is actually CubeMetadataUpgradeV1, - * that upgrades metadata store from v1(prior kylin 0.7) to v2. - * the major difference is that we split cube desc to cube desc + model desc - * - * In 2.0 there is a CubeMetadataUpgradeV2 which is responsible for upgrading the metadata store - * from 1.x to 2.0. The major actions in that is updating cube desc signature and upgrade model desc. - * - * this CubeMetadataUpgradeV3 upgrades metadata store from v2(prior kylin 2.1) to v3 - * the major different is a brand new definition of partial cubes to allow users to select - * cuboids more flexibly. https://issues.apache.org/jira/browse/KYLIN-242 - */ -public class CubeMetadataUpgradeV3 { - - private KylinConfig config = null; - private ResourceStore store; - - private List<String> updatedResources = Lists.newArrayList(); - private List<String> errorMsgs = Lists.newArrayList(); - - private static final Log logger = LogFactory.getLog(CubeMetadataUpgradeV3.class); - - public CubeMetadataUpgradeV3(String newMetadataUrl) { - KylinConfig.destoryInstance(); - System.setProperty(KylinConfig.KYLIN_CONF, newMetadataUrl); - KylinConfig.getInstanceFromEnv().setMetadataUrl(newMetadataUrl); - - config = KylinConfig.getInstanceFromEnv(); - store = getStore(); - } - - public void upgrade() { - - upgradeCubeDesc(); - verify(); - } - - public void verify() { - MetadataManager.clearCache(); - MetadataManager.getInstance(config); - CubeDescManager.clearCache(); - CubeDescManager.getInstance(config); - CubeManager.clearCache(); - CubeManager.getInstance(config); - ProjectManager.clearCache(); - ProjectManager.getInstance(config); - //cleanup(); - } - - private List<String> listResourceStore(String pathRoot) { - List<String> paths = null; - try { - paths = store.collectResourceRecursively(pathRoot, MetadataConstants.FILE_SURFIX); - } catch (IOException e1) { - e1.printStackTrace(); - errorMsgs.add("Get IOException when scan resource store at: " + ResourceStore.CUBE_DESC_RESOURCE_ROOT); - } - - return paths; - } - - private void upgradeCubeDesc() { - logger.info("Reloading Cube Metadata from folder " + store.getReadableResourcePath(ResourceStore.CUBE_DESC_RESOURCE_ROOT)); - - List<String> paths = listResourceStore(ResourceStore.CUBE_DESC_RESOURCE_ROOT); - for (String path : paths) { - - try { - CubeDescUpgraderV3 upgrade = new CubeDescUpgraderV3(path); - CubeDesc ndesc = upgrade.upgrade(); - ndesc.setSignature(ndesc.calculateSignature()); - - getStore().putResource(ndesc.getResourcePath(), ndesc, CubeDescManager.CUBE_DESC_SERIALIZER); - updatedResources.add(ndesc.getResourcePath()); - } catch (IOException e) { - e.printStackTrace(); - errorMsgs.add("Upgrade CubeDesc at '" + path + "' failed: " + e.getLocalizedMessage()); - } - } - - } - - private ResourceStore getStore() { - return ResourceStore.getStore(config); - } - - public static void main(String[] args) { - - if (!(args != null && (args.length == 1 || args.length == 2))) { - System.out.println("Usage: java CubeMetadataUpgrade <metadata_export_folder> <verify>; e.g, /export/kylin/meta "); - return; - } - - String exportFolder = args[0]; - boolean verify = false; - if (args.length == 2 && "verify".equals(args[1])) { - System.out.println("Only verify the metadata in folder " + exportFolder); - verify = true; - } - - CubeMetadataUpgradeV3 instance = null; - if (verify) { - instance = new CubeMetadataUpgradeV3(exportFolder); - instance.verify(); - } else { - File oldMetaFolder = new File(exportFolder); - if (!oldMetaFolder.exists()) { - System.out.println("Provided folder doesn't exist: '" + exportFolder + "'"); - return; - } - - if (!oldMetaFolder.isDirectory()) { - System.out.println("Provided folder is not a directory: '" + exportFolder + "'"); - return; - } - - String newMetadataUrl = oldMetaFolder.getAbsolutePath() + "_v3";//upgrades metadata store to v3 format - try { - FileUtils.deleteDirectory(new File(newMetadataUrl)); - FileUtils.copyDirectory(oldMetaFolder, new File(newMetadataUrl)); - } catch (IOException e) { - e.printStackTrace(); - } - - instance = new CubeMetadataUpgradeV3(newMetadataUrl); - instance.upgrade(); - logger.info("================================================================="); - logger.info("Run CubeMetadataUpgrade completed;"); - - } - - logger.info("================================================================="); - if (instance.errorMsgs.size() > 0) { - logger.info("Here are the error/warning messages, you may need check:"); - for (String s : instance.errorMsgs) { - logger.warn(s); - } - } else { - logger.info("No error or warning messages; The migration is success."); - } - } -} http://git-wip-us.apache.org/repos/asf/kylin/blob/2512fb6f/core-cube/src/main/java/org/apache/kylin/cube/upgrade/common/CubeMetadataUpgrade.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/upgrade/common/CubeMetadataUpgrade.java b/core-cube/src/main/java/org/apache/kylin/cube/upgrade/common/CubeMetadataUpgrade.java new file mode 100644 index 0000000..6f80410 --- /dev/null +++ b/core-cube/src/main/java/org/apache/kylin/cube/upgrade/common/CubeMetadataUpgrade.java @@ -0,0 +1,146 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kylin.cube.upgrade.common; + +import java.io.File; +import java.io.IOException; +import java.lang.reflect.InvocationTargetException; +import java.util.List; + +import org.apache.commons.io.FileUtils; +import org.apache.kylin.common.KylinConfig; +import org.apache.kylin.common.persistence.ResourceStore; +import org.apache.kylin.cube.CubeDescManager; +import org.apache.kylin.cube.CubeManager; +import org.apache.kylin.metadata.MetadataConstants; +import org.apache.kylin.metadata.MetadataManager; +import org.apache.kylin.metadata.project.ProjectManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.collect.Lists; + +public abstract class CubeMetadataUpgrade { + private static final Logger logger = LoggerFactory.getLogger(CubeMetadataUpgrade.class); + + protected KylinConfig config = null; + protected ResourceStore store; + protected List<String> updatedResources = Lists.newArrayList(); + protected List<String> errorMsgs = Lists.newArrayList(); + + public CubeMetadataUpgrade(String newMetadataUrl) { + KylinConfig.destoryInstance(); + config = KylinConfig.createInstanceFromUri(newMetadataUrl); + store = ResourceStore.getStore(config); + } + + protected List<String> listResourceStore(String pathRoot) { + List<String> paths = null; + try { + paths = store.collectResourceRecursively(pathRoot, MetadataConstants.FILE_SURFIX); + } catch (IOException e1) { + logger.error("error", e1); + errorMsgs.add("Get IOException when scan resource store at: " + ResourceStore.CUBE_DESC_RESOURCE_ROOT); + } + + return paths; + } + + public void clear() { + MetadataManager.clearCache(); + CubeDescManager.clearCache(); + CubeManager.clearCache(); + ProjectManager.clearCache(); + } + + public void verify() { + logger.info("================================================================="); + logger.info("The changes are applied, now it's time to verify the new metadata store by reloading all metadata:"); + logger.info("================================================================="); + MetadataManager.clearCache(); + MetadataManager.getInstance(config); + CubeDescManager.clearCache(); + CubeDescManager.getInstance(config); + CubeManager.clearCache(); + CubeManager.getInstance(config); + ProjectManager.clearCache(); + ProjectManager.getInstance(config); + //cleanup(); + } + + public abstract void upgradeNonCompatibleMeta(); + + public abstract void upgradeCompatibleMeta(); + + public static void upgradeOrVerify(Class upgradeClass, String[] args, boolean firstStepInChain, boolean lastStepInChain) throws NoSuchMethodException, IllegalAccessException, InvocationTargetException, InstantiationException { + + if (!(args != null && (args.length == 1))) { + System.out.println("Usage: java CubeMetadataUpgrade <metadata_export_folder>"); + System.out.println(", where metadata_export_folder is the folder containing your current metadata's dump (Upgrade program will not modify it directly, relax."); + return; + } + + String currentMetaDumpFolderPath = args[0]; + CubeMetadataUpgrade instance; + + File currentMetaDumpFolder = new File(currentMetaDumpFolderPath); + if (!currentMetaDumpFolder.exists()) { + System.out.println("Provided folder doesn't exist: '" + currentMetaDumpFolderPath + "'"); + return; + } + + if (!currentMetaDumpFolder.isDirectory()) { + System.out.println("Provided folder is not a directory: '" + currentMetaDumpFolderPath + "'"); + return; + } + + String newMetadataUrl; + if (firstStepInChain) { + newMetadataUrl = currentMetaDumpFolder.getAbsolutePath() + "_workspace";//upgrades metadata store in a copy named xx_workspace + try { + FileUtils.deleteDirectory(new File(newMetadataUrl)); + FileUtils.copyDirectory(currentMetaDumpFolder, new File(newMetadataUrl)); + } catch (IOException e) { + throw new RuntimeException(e); + } + } else { + newMetadataUrl = currentMetaDumpFolder.getAbsolutePath(); + } + + instance = (CubeMetadataUpgrade) upgradeClass.getConstructor(String.class).newInstance(newMetadataUrl); + instance.upgradeNonCompatibleMeta(); + logger.info("================================================================="); + logger.info("Run {} completed", upgradeClass.toString()); + logger.info("================================================================="); + if (instance.errorMsgs.size() > 0) { + logger.info("Here are the error/warning messages, you may need check:"); + for (String s : instance.errorMsgs) { + logger.error(s); + } + } else { + logger.info("No error or warning messages; The migration is success."); + } + + if (lastStepInChain) { + instance.upgradeCompatibleMeta(); + instance.verify(); + } + } + +} http://git-wip-us.apache.org/repos/asf/kylin/blob/2512fb6f/core-cube/src/main/java/org/apache/kylin/cube/upgrade/common/MetadataVersionRefresher.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/upgrade/common/MetadataVersionRefresher.java b/core-cube/src/main/java/org/apache/kylin/cube/upgrade/common/MetadataVersionRefresher.java new file mode 100644 index 0000000..511edb8 --- /dev/null +++ b/core-cube/src/main/java/org/apache/kylin/cube/upgrade/common/MetadataVersionRefresher.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kylin.cube.upgrade.common; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.apache.kylin.common.KylinVersion; +import org.apache.kylin.common.persistence.ResourceStore; +import org.apache.kylin.metadata.MetadataConstants; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.common.collect.Lists; + +/** + * After upgrade a metadata store to current version, + * we'll label every RootPersistentEntity's version as KylinVersion.getCurrentVersion() + */ +public class MetadataVersionRefresher { + + private static final Logger logger = LoggerFactory.getLogger(MetadataVersionRefresher.class); + + private ResourceStore store; + + public MetadataVersionRefresher(ResourceStore resourceStore) { + this.store = resourceStore; + } + + public void refresh() throws IOException { + ObjectMapper mapper = new ObjectMapper(); + mapper.configure(SerializationFeature.INDENT_OUTPUT, true); + + List<String> all = Lists.newArrayList(); + collectFiles(this.store, "/", all); + + for (String path : all) { + if (path.endsWith(MetadataConstants.FILE_SURFIX) && !(path.startsWith(ResourceStore.DICT_RESOURCE_ROOT) || path.startsWith(ResourceStore.SNAPSHOT_RESOURCE_ROOT))) { + logger.info("Updating metadata version of path {}", path); + ObjectNode objectNode = (ObjectNode) mapper.readTree(this.store.getResource(path).inputStream); + objectNode.put("version", KylinVersion.getCurrentVersion()); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + mapper.writeValue(baos, objectNode); + ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray()); + this.store.putResource(path, bais, System.currentTimeMillis()); + } + } + } + + public static void collectFiles(ResourceStore src, String path, List<String> ret) throws IOException { + ArrayList<String> children = src.listResources(path); + + // case of resource (not a folder) + if (children == null) { + ret.add(path); + } + // case of folder + else { + for (String child : children) { + collectFiles(src, child, ret); + } + } + } +} http://git-wip-us.apache.org/repos/asf/kylin/blob/2512fb6f/core-cube/src/main/java/org/apache/kylin/cube/upgrade/entry/CubeMetadataUpgradeEntry_v_1_5_0.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/upgrade/entry/CubeMetadataUpgradeEntry_v_1_5_0.java b/core-cube/src/main/java/org/apache/kylin/cube/upgrade/entry/CubeMetadataUpgradeEntry_v_1_5_0.java new file mode 100644 index 0000000..b4a0286 --- /dev/null +++ b/core-cube/src/main/java/org/apache/kylin/cube/upgrade/entry/CubeMetadataUpgradeEntry_v_1_5_0.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kylin.cube.upgrade.entry; + +import org.apache.kylin.cube.upgrade.V1_5_0.CubeMetadataUpgrade_v_1_5_0; +import org.apache.kylin.cube.upgrade.v1_4_0.CubeMetadataUpgrade_v_1_4_0; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class CubeMetadataUpgradeEntry_v_1_5_0 { + private static final Logger logger = LoggerFactory.getLogger(CubeMetadataUpgradeEntry_v_1_5_0.class); + + public static void main(String[] args) { + if (!(args != null && (args.length == 1))) { + System.out.println("Usage: java CubeMetadataUpgradeEntry_v_1_5_0 <metadata_export_folder>"); + System.out.println(", where metadata_export_folder is the folder containing your current metadata's dump (Upgrade program will not modify it directly, relax."); + return; + } + + try { + CubeMetadataUpgrade_v_1_4_0.upgradeOrVerify(CubeMetadataUpgrade_v_1_4_0.class, args, true, false); + CubeMetadataUpgrade_v_1_5_0.upgradeOrVerify(CubeMetadataUpgrade_v_1_5_0.class, new String[] { args[0] + "_workspace" }, false, true); + } catch (Exception e) { + logger.error("something went wrong when upgrading, suggest to roll back metadata", e); + return; + } + + logger.info("The metadata upgrade is complete locally. You need to upload the metadata to you actual metadata store to verify locally. You need to upload the metadata to you actual metadata store to verify."); + } +} http://git-wip-us.apache.org/repos/asf/kylin/blob/2512fb6f/core-cube/src/main/java/org/apache/kylin/cube/upgrade/v1_4_0/CubeMetadataUpgrade_v_1_4_0.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/upgrade/v1_4_0/CubeMetadataUpgrade_v_1_4_0.java b/core-cube/src/main/java/org/apache/kylin/cube/upgrade/v1_4_0/CubeMetadataUpgrade_v_1_4_0.java new file mode 100644 index 0000000..04f09d8 --- /dev/null +++ b/core-cube/src/main/java/org/apache/kylin/cube/upgrade/v1_4_0/CubeMetadataUpgrade_v_1_4_0.java @@ -0,0 +1,181 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kylin.cube.upgrade.v1_4_0; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.kylin.common.persistence.JsonSerializer; +import org.apache.kylin.common.persistence.ResourceStore; +import org.apache.kylin.common.persistence.Serializer; +import org.apache.kylin.cube.model.v1_4_0.CubeDesc; +import org.apache.kylin.cube.model.v1_4_0.DimensionDesc; +import org.apache.kylin.cube.upgrade.common.CubeMetadataUpgrade; +import org.apache.kylin.metadata.MetadataManager; +import org.apache.kylin.metadata.model.DataModelDesc; +import org.apache.kylin.metadata.model.MeasureDesc; +import org.apache.kylin.metadata.model.ModelDimensionDesc; +import org.apache.kylin.metadata.model.PartitionDesc; +import org.apache.kylin.metadata.model.TblColRef; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; + +/** + * please check doc on CubeMetadataUpgrade_v_1_5_0 + */ +public class CubeMetadataUpgrade_v_1_4_0 extends CubeMetadataUpgrade { + + private static final Logger logger = LoggerFactory.getLogger(CubeMetadataUpgrade_v_1_4_0.class); + private static final Serializer<CubeDesc> oldCubeDescSerializer = new JsonSerializer<>(CubeDesc.class); + + private List<String> updatedResources = Lists.newArrayList(); + + public CubeMetadataUpgrade_v_1_4_0(String newMetadataUrl) { + super(newMetadataUrl); + } + + public void upgradeNonCompatibleMeta() { + dowork(); + } + + public void upgradeCompatibleMeta() { + //do nothing + } + + + + private CubeDesc loadOldCubeDesc(String path) { + CubeDesc ndesc = null; + try { + ndesc = store.getResource(path, CubeDesc.class, oldCubeDescSerializer); + } catch (IOException e) { + throw new RuntimeException("failed to load old cubedesc at " + path); + } + + if (StringUtils.isBlank(ndesc.getName())) { + throw new IllegalStateException("CubeDesc name must not be blank"); + } + + return ndesc; + } + + private DataModelDesc getDataModelDesc(String modelName) { + MetadataManager.clearCache(); + return MetadataManager.getInstance(config).getDataModelDesc(modelName); + } + + public void dowork() { + List<String> paths = listResourceStore(ResourceStore.CUBE_DESC_RESOURCE_ROOT); + for (String path : paths) { + logger.info("CubeMetadataUpgrade_v_1_4_0 handling in dowork {}", path); + CubeDesc cubeDesc = loadOldCubeDesc(path); + cubeDesc.init(config, MetadataManager.getInstance(config).getAllTablesMap()); + + upgradeDataModelDesc(cubeDesc); + upgradeCubeDesc(cubeDesc); + } + } + + private void upgradeDataModelDesc(CubeDesc cubeDesc) { + boolean upgrade = false; + + DataModelDesc modelDesc = getDataModelDesc(cubeDesc.getModelName()); + + try { + if (modelDesc != null && modelDesc.getDimensions() == null && modelDesc.getMetrics() == null) { + List<DimensionDesc> cubeDimDescList = cubeDesc.getDimensions(); + if (!CollectionUtils.isEmpty(cubeDimDescList)) { + Map<String, HashSet<String>> modelDimMap = Maps.newHashMap(); + for (DimensionDesc cubeDimDesc : cubeDimDescList) { + if (!modelDimMap.containsKey(cubeDimDesc.getTable())) { + modelDimMap.put(cubeDimDesc.getTable(), new HashSet<String>()); + } + modelDimMap.get(cubeDimDesc.getTable()).addAll(Lists.newArrayList(cubeDimDesc.getDerived() != null ? cubeDimDesc.getDerived() : cubeDimDesc.getColumn())); + } + + List<ModelDimensionDesc> modelDimDescList = Lists.newArrayListWithCapacity(modelDimMap.size()); + for (Map.Entry<String, HashSet<String>> modelDimEntry : modelDimMap.entrySet()) { + ModelDimensionDesc dimDesc = new ModelDimensionDesc(); + dimDesc.setTable(modelDimEntry.getKey()); + String[] columns = new String[modelDimEntry.getValue().size()]; + columns = modelDimEntry.getValue().toArray(columns); + dimDesc.setColumns(columns); + modelDimDescList.add(dimDesc); + } + ModelDimensionDesc.capicalizeStrings(modelDimDescList); + modelDesc.setDimensions(modelDimDescList); + upgrade = true; + } + + List<MeasureDesc> cubeMeasDescList = cubeDesc.getMeasures(); + if (!CollectionUtils.isEmpty(cubeDimDescList)) { + ArrayList<String> metrics = Lists.newArrayListWithExpectedSize(cubeMeasDescList.size()); + for (MeasureDesc cubeMeasDesc : cubeMeasDescList) { + for (TblColRef tblColRef : cubeMeasDesc.getFunction().getParameter().getColRefs()) { + metrics.add(tblColRef.getName()); + } + } + String[] metricsArray = new String[metrics.size()]; + modelDesc.setMetrics(metrics.toArray(metricsArray)); + upgrade = true; + } + } + + if (upgrade) { + store.putResource(modelDesc.getResourcePath(), modelDesc, MetadataManager.MODELDESC_SERIALIZER); + updatedResources.add(modelDesc.getResourcePath()); + } + } catch (Exception e) { + logger.error("error", e); + + errorMsgs.add("upgradeDataModelDesc [" + modelDesc.getName() + "] failed: " + e.getLocalizedMessage()); + } + } + + private void upgradeCubeDesc(CubeDesc cubeDesc) { + try { + + DataModelDesc modelDesc = getDataModelDesc(cubeDesc.getModelName()); + + PartitionDesc modelPartDesc = modelDesc.getPartitionDesc(); + if (cubeDesc.getPartitionDateStart() == 0 && modelPartDesc.getPartitionDateStart() != 0) { + cubeDesc.setPartitionDateStart(modelPartDesc.getPartitionDateStart()); + } + + // String calculatedSign = cubeDesc.calculateSignature(); + // cubeDesc.setSignature(calculatedSign); + + store.putResource(cubeDesc.getResourcePath(), cubeDesc, oldCubeDescSerializer); + updatedResources.add(cubeDesc.getResourcePath()); + + } catch (Exception e) { + logger.error("error", e); + errorMsgs.add("upgradeCubeDesc [" + cubeDesc.getName() + "] failed: " + e.getLocalizedMessage()); + } + } + +} http://git-wip-us.apache.org/repos/asf/kylin/blob/2512fb6f/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java b/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java index 9f2a934..3aa3810 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java @@ -114,6 +114,10 @@ public class MetadataManager { init(config); } + public static String concatDataModelResourcePath(String modelName) { + return ResourceStore.DATA_MODEL_DESC_RESOURCE_ROOT + "/" + modelName + MetadataConstants.FILE_SURFIX; + } + /** * Tell MetadataManager that the instance has changed. The cube info will * be stored Reload the cube desc and source table A broadcast must be sent @@ -208,7 +212,7 @@ public class MetadataManager { this.srcTableMap = new CaseInsensitiveStringCache<TableDesc>(config, Broadcaster.TYPE.TABLE); this.srcTableExdMap = new CaseInsensitiveStringCache<Map<String, String>>(config, Broadcaster.TYPE.TABLE); this.dataModelDescMap = new CaseInsensitiveStringCache<DataModelDesc>(config, Broadcaster.TYPE.DATA_MODEL); - + reloadAllSourceTable(); reloadAllSourceTableExd(); reloadAllDataModel(); @@ -238,7 +242,7 @@ public class MetadataManager { logger.warn("Failed to get table exd info from " + path); return null; } - + InputStream is = res.inputStream; try { @@ -343,8 +347,8 @@ public class MetadataManager { } public boolean isTableInModel(String tableName, String projectName) throws IOException { - for(DataModelDesc modelDesc : getModels(projectName)) { - if(modelDesc.getAllTables().contains(tableName.toUpperCase())) { + for (DataModelDesc modelDesc : getModels(projectName)) { + if (modelDesc.getAllTables().contains(tableName.toUpperCase())) { return true; } } @@ -352,8 +356,8 @@ public class MetadataManager { } public boolean isTableInAnyModel(String tableName) { - for(DataModelDesc modelDesc : getModels()) { - if(modelDesc.getAllTables().contains(tableName.toUpperCase())){ + for (DataModelDesc modelDesc : getModels()) { + if (modelDesc.getAllTables().contains(tableName.toUpperCase())) { return true; } } http://git-wip-us.apache.org/repos/asf/kylin/blob/2512fb6f/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java index 1647707..28e26fa 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java @@ -58,13 +58,14 @@ public class DataModelDesc extends RootPersistentEntity { private LookupDesc[] lookups; @JsonProperty("dimensions") - private List<DimensionDesc> dimensions; + private List<ModelDimensionDesc> dimensions; @JsonProperty("metrics") private String[] metrics; @JsonProperty("filter_condition") private String filterCondition; + @JsonProperty("partition_desc") PartitionDesc partitionDesc; @@ -72,6 +73,7 @@ public class DataModelDesc extends RootPersistentEntity { private RealizationCapacity capacity = RealizationCapacity.MEDIUM; private TableDesc factTableDesc; + private List<TableDesc> lookupTableDescs = Lists.newArrayList(); /** @@ -188,7 +190,7 @@ public class DataModelDesc extends RootPersistentEntity { } initJoinColumns(tables); - DimensionDesc.capicalizeStrings(dimensions); + ModelDimensionDesc.capicalizeStrings(dimensions); initPartitionDesc(tables); } @@ -321,7 +323,7 @@ public class DataModelDesc extends RootPersistentEntity { return ResourceStore.DATA_MODEL_DESC_RESOURCE_ROOT + "/" + descName + MetadataConstants.FILE_SURFIX; } - public List<DimensionDesc> getDimensions() { + public List<ModelDimensionDesc> getDimensions() { return dimensions; } @@ -329,7 +331,7 @@ public class DataModelDesc extends RootPersistentEntity { return metrics; } - public void setDimensions(List<DimensionDesc> dimensions) { + public void setDimensions(List<ModelDimensionDesc> dimensions) { this.dimensions = dimensions; } http://git-wip-us.apache.org/repos/asf/kylin/blob/2512fb6f/core-metadata/src/main/java/org/apache/kylin/metadata/model/DimensionDesc.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DimensionDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DimensionDesc.java deleted file mode 100644 index 0892fb0..0000000 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DimensionDesc.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package org.apache.kylin.metadata.model; - -import java.util.List; - -import org.apache.kylin.common.util.StringUtil; - -import com.fasterxml.jackson.annotation.JsonAutoDetect; -import com.fasterxml.jackson.annotation.JsonProperty; - -/** - */ -@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.NONE, getterVisibility = JsonAutoDetect.Visibility.NONE, isGetterVisibility = JsonAutoDetect.Visibility.NONE, setterVisibility = JsonAutoDetect.Visibility.NONE) -public class DimensionDesc { - @JsonProperty("table") - private String table; - @JsonProperty("columns") - private String[] columns; - - public String getTable() { - return table; - } - - public void setTable(String table) { - this.table = table; - } - - public String[] getColumns() { - return columns; - } - - public void setColumns(String[] columns) { - this.columns = columns; - } - - public static void capicalizeStrings(List<DimensionDesc> dimensions) { - if (dimensions != null) { - for (DimensionDesc dimensionDesc : dimensions) { - dimensionDesc.setTable(dimensionDesc.getTable().toUpperCase()); - if (dimensionDesc.getColumns() != null) { - StringUtil.toUpperCaseArray(dimensionDesc.getColumns(), dimensionDesc.getColumns()); - } - } - } - } - - public static int getColumnCount(List<DimensionDesc> dimensionDescs) { - int count = 0; - for (DimensionDesc dimensionDesc : dimensionDescs) { - if (dimensionDesc.getColumns() != null) { - count += dimensionDesc.getColumns().length; - } - } - return count; - } - -} http://git-wip-us.apache.org/repos/asf/kylin/blob/2512fb6f/core-metadata/src/main/java/org/apache/kylin/metadata/model/ModelDimensionDesc.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ModelDimensionDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ModelDimensionDesc.java new file mode 100644 index 0000000..d196155 --- /dev/null +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ModelDimensionDesc.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package org.apache.kylin.metadata.model; + +import java.util.List; + +import org.apache.kylin.common.util.StringUtil; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + */ +@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.NONE, getterVisibility = JsonAutoDetect.Visibility.NONE, isGetterVisibility = JsonAutoDetect.Visibility.NONE, setterVisibility = JsonAutoDetect.Visibility.NONE) +public class ModelDimensionDesc { + @JsonProperty("table") + private String table; + @JsonProperty("columns") + private String[] columns; + + public String getTable() { + return table; + } + + public void setTable(String table) { + this.table = table; + } + + public String[] getColumns() { + return columns; + } + + public void setColumns(String[] columns) { + this.columns = columns; + } + + public static void capicalizeStrings(List<ModelDimensionDesc> dimensions) { + if (dimensions != null) { + for (ModelDimensionDesc modelDimensionDesc : dimensions) { + modelDimensionDesc.setTable(modelDimensionDesc.getTable().toUpperCase()); + if (modelDimensionDesc.getColumns() != null) { + StringUtil.toUpperCaseArray(modelDimensionDesc.getColumns(), modelDimensionDesc.getColumns()); + } + } + } + } + + public static int getColumnCount(List<ModelDimensionDesc> modelDimensionDescs) { + int count = 0; + for (ModelDimensionDesc modelDimensionDesc : modelDimensionDescs) { + if (modelDimensionDesc.getColumns() != null) { + count += modelDimensionDesc.getColumns().length; + } + } + return count; + } + +} http://git-wip-us.apache.org/repos/asf/kylin/blob/2512fb6f/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java index 2556743..1738c27 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java @@ -48,7 +48,8 @@ public class PartitionDesc { private String partitionTimeColumn; @JsonProperty("partition_date_start") - private long partitionDateStart = 0L; + private long partitionDateStart = 0L;//Deprecated + @JsonProperty("partition_date_format") private String partitionDateFormat = DateFormat.DEFAULT_DATE_PATTERN; @@ -57,6 +58,7 @@ public class PartitionDesc { @JsonProperty("partition_type") private PartitionType partitionType = PartitionType.APPEND; + @JsonProperty("partition_condition_builder") private String partitionConditionBuilderClz = DefaultPartitionConditionBuilder.class.getName(); http://git-wip-us.apache.org/repos/asf/kylin/blob/2512fb6f/examples/test_case_data/localmeta/cube/ssb.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/cube/ssb.json b/examples/test_case_data/localmeta/cube/ssb.json index 879754e..84d77dc 100644 --- a/examples/test_case_data/localmeta/cube/ssb.json +++ b/examples/test_case_data/localmeta/cube/ssb.json @@ -1,6 +1,5 @@ { "uuid" : "70a9f288-3c01-4745-a04b-5641e82d6c69", - "version" : "2.1", "name" : "ssb", "owner" : "ADMIN", "cost" : 50, http://git-wip-us.apache.org/repos/asf/kylin/blob/2512fb6f/examples/test_case_data/localmeta/cube/test_kylin_cube_with_slr_empty.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/cube/test_kylin_cube_with_slr_empty.json b/examples/test_case_data/localmeta/cube/test_kylin_cube_with_slr_empty.json index 19c625c..90488d7 100644 --- a/examples/test_case_data/localmeta/cube/test_kylin_cube_with_slr_empty.json +++ b/examples/test_case_data/localmeta/cube/test_kylin_cube_with_slr_empty.json @@ -1,6 +1,5 @@ { "uuid" : "1eaca32a-a33e-4b69-83dd-0bb8b1f8c53b", - "last_modified" : 0, "name" : "test_kylin_cube_with_slr_empty", "owner" : null, http://git-wip-us.apache.org/repos/asf/kylin/blob/2512fb6f/examples/test_case_data/localmeta/cube_desc/ssb.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/cube_desc/ssb.json b/examples/test_case_data/localmeta/cube_desc/ssb.json index 7fa0e9a..a650fd3 100644 --- a/examples/test_case_data/localmeta/cube_desc/ssb.json +++ b/examples/test_case_data/localmeta/cube_desc/ssb.json @@ -1,6 +1,5 @@ { "uuid" : "5c44df30-daec-486e-af90-927bf7851057", - "version" : "2.1", "name" : "ssb", "description" : "", "dimensions" : [ { http://git-wip-us.apache.org/repos/asf/kylin/blob/2512fb6f/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIDesc.java ---------------------------------------------------------------------- diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIDesc.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIDesc.java index cf57519..30b5d0a 100644 --- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIDesc.java +++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIDesc.java @@ -37,7 +37,7 @@ import org.apache.kylin.metadata.MetadataConstants; import org.apache.kylin.metadata.MetadataManager; import org.apache.kylin.metadata.model.ColumnDesc; import org.apache.kylin.metadata.model.DataModelDesc; -import org.apache.kylin.metadata.model.DimensionDesc; +import org.apache.kylin.metadata.model.ModelDimensionDesc; import org.apache.kylin.metadata.model.FunctionDesc; import org.apache.kylin.metadata.model.IEngineAware; import org.apache.kylin.metadata.model.IStorageAware; @@ -79,9 +79,9 @@ public class IIDesc extends RootPersistentEntity { @JsonProperty("timestamp_dimension") private String timestampDimension; @JsonProperty("bitmap_dimensions") - private List<DimensionDesc> bitmapDimensions = Collections.emptyList(); + private List<ModelDimensionDesc> bitmapDimensions = Collections.emptyList(); @JsonProperty("value_dimensions") - private List<DimensionDesc> valueDimensions; + private List<ModelDimensionDesc> valueDimensions; @JsonProperty("metrics") private String[] metricNames; @JsonProperty("sharding") @@ -126,17 +126,17 @@ public class IIDesc extends RootPersistentEntity { timestampDimension = timestampDimension.toUpperCase(); // capitalize - DimensionDesc.capicalizeStrings(bitmapDimensions); - DimensionDesc.capicalizeStrings(valueDimensions); + ModelDimensionDesc.capicalizeStrings(bitmapDimensions); + ModelDimensionDesc.capicalizeStrings(valueDimensions); StringUtil.toUpperCaseArray(metricNames, metricNames); // retrieve all columns and all tables, and make available measure to ii HashSet<String> allTableNames = Sets.newHashSet(); measureDescs = Lists.newArrayList(); measureDescs.add(makeCountMeasure()); - for (DimensionDesc dimensionDesc : Iterables.concat(bitmapDimensions, valueDimensions)) { - TableDesc tableDesc = this.getTableDesc(dimensionDesc.getTable()); - for (String column : dimensionDesc.getColumns()) { + for (ModelDimensionDesc modelDimensionDesc : Iterables.concat(bitmapDimensions, valueDimensions)) { + TableDesc tableDesc = this.getTableDesc(modelDimensionDesc.getTable()); + for (String column : modelDimensionDesc.getColumns()) { ColumnDesc columnDesc = tableDesc.findColumnByName(column); TblColRef tcr = new TblColRef(columnDesc); allColumns.add(tcr); @@ -163,8 +163,8 @@ public class IIDesc extends RootPersistentEntity { } // indexing for each type of columns - bitmapCols = new int[DimensionDesc.getColumnCount(bitmapDimensions)]; - valueCols = new int[DimensionDesc.getColumnCount(valueDimensions)]; + bitmapCols = new int[ModelDimensionDesc.getColumnCount(bitmapDimensions)]; + valueCols = new int[ModelDimensionDesc.getColumnCount(valueDimensions)]; metricsCols = new int[metricNames.length]; metricsColSet = new BitSet(this.getTableDesc(this.getFactTableName()).getColumnCount()); http://git-wip-us.apache.org/repos/asf/kylin/blob/2512fb6f/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/SandboxMetastoreCLI.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/SandboxMetastoreCLI.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/SandboxMetastoreCLI.java index 7b3b698..30889ab 100644 --- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/SandboxMetastoreCLI.java +++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/SandboxMetastoreCLI.java @@ -52,7 +52,9 @@ public class SandboxMetastoreCLI { return; } - if ("download".equalsIgnoreCase(args[0])) { + if ("reset".equalsIgnoreCase(args[0])) { + ResourceTool.main(new String[] { "reset" }); + }else if ("download".equalsIgnoreCase(args[0])) { ResourceTool.main(new String[] { "download", args[1] }); } else if ("upload".equalsIgnoreCase(args[0])) { ResourceTool.main(new String[] { "upload", args[1] });
