KYLIN-2198 Add a framework to allow major changes in DimensionEncoding
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e1acc419 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e1acc419 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e1acc419 Branch: refs/heads/yang21 Commit: e1acc4192a982f897489f52d1bbc836a5c207da6 Parents: c0c0814 Author: Hongbin Ma <mahong...@apache.org> Authored: Wed Nov 16 14:47:53 2016 +0800 Committer: Hongbin Ma <mahong...@apache.org> Committed: Thu Nov 17 11:11:34 2016 +0800 ---------------------------------------------------------------------- .../apache/kylin/common/util/JacksonBean.java | 55 +++++++++++ .../apache/kylin/common/util/JacksonTest.java | 39 ++++++++ .../org/apache/kylin/cube/kv/CubeDimEncMap.java | 2 +- .../apache/kylin/cube/model/RowKeyColDesc.java | 13 ++- .../dimension/DimensionEncodingFactory.java | 97 +++++++++++++++----- .../kylin/measure/topn/TopNMeasureType.java | 12 ++- .../kylin/rest/controller/CubeController.java | 12 +-- 7 files changed, 198 insertions(+), 32 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/e1acc419/core-common/src/test/java/org/apache/kylin/common/util/JacksonBean.java ---------------------------------------------------------------------- diff --git a/core-common/src/test/java/org/apache/kylin/common/util/JacksonBean.java b/core-common/src/test/java/org/apache/kylin/common/util/JacksonBean.java new file mode 100644 index 0000000..42357f2 --- /dev/null +++ b/core-common/src/test/java/org/apache/kylin/common/util/JacksonBean.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kylin.common.util; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; + +@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE) +public class JacksonBean { + + @JsonProperty("a") + private String a; + @JsonProperty("b") + @JsonInclude(JsonInclude.Include.NON_NULL) + private int b; + + public String getA() { + return a; + } + + public void setA(String a) { + this.a = a; + } + + public int getB() { + return b; + } + + public void setB(int b) { + this.b = b; + } + + @Override + public String toString() { + return "JacksonBean{" + "a='" + a + '\'' + ", b=" + b + '}'; + } +} http://git-wip-us.apache.org/repos/asf/kylin/blob/e1acc419/core-common/src/test/java/org/apache/kylin/common/util/JacksonTest.java ---------------------------------------------------------------------- diff --git a/core-common/src/test/java/org/apache/kylin/common/util/JacksonTest.java b/core-common/src/test/java/org/apache/kylin/common/util/JacksonTest.java new file mode 100644 index 0000000..81be7eb --- /dev/null +++ b/core-common/src/test/java/org/apache/kylin/common/util/JacksonTest.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kylin.common.util; + +import java.io.IOException; + +import org.junit.Test; + +public class JacksonTest { + @Test + public void foo() throws IOException { + JacksonBean bean = new JacksonBean(); + bean.setA("valuea"); + + String s = JsonUtil.writeValueAsString(bean); + System.out.println(s); + + JacksonBean desBean = (JacksonBean) JsonUtil.readValue("{\"a\":\"valuea\"}", JacksonBean.class); + String x2 = JsonUtil.writeValueAsString(desBean); + System.out.println(desBean); + System.out.println(x2); + } +} http://git-wip-us.apache.org/repos/asf/kylin/blob/e1acc419/core-cube/src/main/java/org/apache/kylin/cube/kv/CubeDimEncMap.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/kv/CubeDimEncMap.java b/core-cube/src/main/java/org/apache/kylin/cube/kv/CubeDimEncMap.java index f588986..a4d2d6f 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/kv/CubeDimEncMap.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/kv/CubeDimEncMap.java @@ -72,7 +72,7 @@ public class CubeDimEncMap implements IDimensionEncodingMap { } } else { // normal case - result = DimensionEncodingFactory.create(colDesc.getEncodingName(), colDesc.getEncodingArgs()); + result = DimensionEncodingFactory.create(colDesc.getEncodingName(), colDesc.getEncodingArgs(), colDesc.getEncodingVersion()); } encMap.put(col, result); } http://git-wip-us.apache.org/repos/asf/kylin/blob/e1acc419/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java index 12c4dfc..9b32d8c 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java @@ -47,6 +47,9 @@ public class RowKeyColDesc { private String column; @JsonProperty("encoding") private String encoding; + @JsonProperty("encoding_version") + @JsonInclude(JsonInclude.Include.NON_DEFAULT) + private int encodingVersion = 1; @JsonProperty("isShardBy") private boolean isShardBy;//usually it is ultra high cardinality column, shard by such column can reduce the agg cache for each shard @JsonProperty("index") @@ -72,7 +75,7 @@ public class RowKeyColDesc { encodingName = (String) encodingConf[0]; encodingArgs = (String[]) encodingConf[1]; - if (!DimensionEncodingFactory.isVaildEncoding(this.encodingName)) + if (!DimensionEncodingFactory.isValidEncoding(this.encodingName)) throw new IllegalArgumentException("Not supported row key col encoding: '" + this.encoding + "'"); // convert date/time dictionary on date/time column to DimensionEncoding implicitly @@ -144,6 +147,14 @@ public class RowKeyColDesc { this.index = index; } + public int getEncodingVersion() { + return encodingVersion; + } + + public void setEncodingVersion(int encodingVersion) { + this.encodingVersion = encodingVersion; + } + @Override public String toString() { return Objects.toStringHelper(this).add("column", column).add("encoding", encoding).toString(); http://git-wip-us.apache.org/repos/asf/kylin/blob/e1acc419/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncodingFactory.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncodingFactory.java b/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncodingFactory.java index 27bebd7..242e003 100644 --- a/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncodingFactory.java +++ b/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncodingFactory.java @@ -18,30 +18,47 @@ package org.apache.kylin.dimension; +import java.util.Arrays; import java.util.Map; -import java.util.Set; -import java.util.TreeSet; + +import javax.annotation.Nullable; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.util.ClassUtil; +import org.apache.kylin.common.util.Pair; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.google.common.base.Predicate; +import com.google.common.collect.Iterables; import com.google.common.collect.Maps; -import com.google.common.collect.Sets; public abstract class DimensionEncodingFactory { private static final Logger logger = LoggerFactory.getLogger(DimensionEncodingFactory.class); - private static Map<String, DimensionEncodingFactory> factoryMap; + private static Map<Pair<String, Integer>, DimensionEncodingFactory> factoryMap; + + /** + * If a bug found in a DimEnc will cause different cube outputs, + * we'll have to increase the version number of DimEnc, in order + * to distinguish current version with prior version. + * <p> + * The default version applys to all existing legacy DimEncs + */ + protected int getCurrentVersion() { + return 1; + } - /** Create a DimensionEncoding instance, with inputs corresponding to RowKeyColDesc.encodingName and RowKeyColDesc.encodingArgs. */ - public static DimensionEncoding create(String encodingName, String[] args) { + /** + * Create a DimensionEncoding instance, with inputs corresponding to RowKeyColDesc.encodingName and RowKeyColDesc.encodingArgs. + */ + public static DimensionEncoding create(String encodingName, String[] args, int version) { + logger.debug("Encoding Name : {}, args : {}, version {}", encodingName, Arrays.toString(args), version); if (factoryMap == null) initFactoryMap(); - DimensionEncodingFactory factory = factoryMap.get(encodingName); + DimensionEncodingFactory factory = factoryMap.get(Pair.newPair(encodingName, version)); if (factory == null) { throw new IllegalArgumentException("Unknown dimension encoding name " + encodingName // + " (note '" + DictionaryDimEnc.ENCODING_NAME + "' is not handled by factory)"); @@ -50,42 +67,72 @@ public abstract class DimensionEncodingFactory { return factory.createDimensionEncoding(encodingName, args); } - public static Set<String> getValidEncodings() { + public static Map<String, Integer> getValidEncodings() { if (factoryMap == null) initFactoryMap(); - TreeSet<String> result = Sets.newTreeSet(); - result.addAll(factoryMap.keySet()); - result.add(DictionaryDimEnc.ENCODING_NAME); + Map<String, Integer> result = Maps.newHashMap(); + for (Pair<String, Integer> p : factoryMap.keySet()) { + result.put(p.getFirst(), p.getSecond()); + } + result.put(DictionaryDimEnc.ENCODING_NAME, 1); return result; } - public static boolean isVaildEncoding(String encodingName) { + public static boolean isValidEncoding(final String encodingName) { if (factoryMap == null) initFactoryMap(); // note dictionary is a special case - return DictionaryDimEnc.ENCODING_NAME.equals(encodingName) || factoryMap.containsKey(encodingName); + return DictionaryDimEnc.ENCODING_NAME.equals(encodingName) || // + Iterables.any(factoryMap.keySet(), new Predicate<Pair<String, Integer>>() { + @Override + public boolean apply(@Nullable Pair<String, Integer> input) { + return input.getFirst().equals(encodingName); + } + }); } private synchronized static void initFactoryMap() { if (factoryMap == null) { - Map<String, DimensionEncodingFactory> map = Maps.newConcurrentMap(); + Map<Pair<String, Integer>, DimensionEncodingFactory> map = Maps.newConcurrentMap(); // built-in encodings, note dictionary is a special case - map.put(FixedLenDimEnc.ENCODING_NAME, new FixedLenDimEnc.Factory()); - map.put(IntDimEnc.ENCODING_NAME, new IntDimEnc.Factory()); - map.put(IntegerDimEnc.ENCODING_NAME, new IntegerDimEnc.Factory()); - map.put(FixedLenHexDimEnc.ENCODING_NAME, new FixedLenHexDimEnc.Factory()); - map.put(DateDimEnc.ENCODING_NAME, new DateDimEnc.Factory()); - map.put(TimeDimEnc.ENCODING_NAME, new TimeDimEnc.Factory()); + { + FixedLenDimEnc.Factory value = new FixedLenDimEnc.Factory(); + map.put(Pair.newPair(FixedLenDimEnc.ENCODING_NAME, value.getCurrentVersion()), value); + } + { + IntDimEnc.Factory value = new IntDimEnc.Factory(); + map.put(Pair.newPair(IntDimEnc.ENCODING_NAME, value.getCurrentVersion()), value); + } + { + IntegerDimEnc.Factory value = new IntegerDimEnc.Factory(); + map.put(Pair.newPair(IntegerDimEnc.ENCODING_NAME, value.getCurrentVersion()), value); + } + { + IntegerDimEncV2.Factory value = new IntegerDimEncV2.Factory(); + map.put(Pair.newPair(IntegerDimEncV2.ENCODING_NAME, value.getCurrentVersion()), value); + } + { + FixedLenHexDimEnc.Factory value = new FixedLenHexDimEnc.Factory(); + map.put(Pair.newPair(FixedLenHexDimEnc.ENCODING_NAME, value.getCurrentVersion()), value); + } + { + DateDimEnc.Factory value = new DateDimEnc.Factory(); + map.put(Pair.newPair(DateDimEnc.ENCODING_NAME, value.getCurrentVersion()), value); + } + { + TimeDimEnc.Factory value = new TimeDimEnc.Factory(); + map.put(Pair.newPair(TimeDimEnc.ENCODING_NAME, value.getCurrentVersion()), value); + } // custom encodings String[] clsNames = KylinConfig.getInstanceFromEnv().getCubeDimensionCustomEncodingFactories(); for (String clsName : clsNames) { try { DimensionEncodingFactory factory = (DimensionEncodingFactory) ClassUtil.newInstance(clsName); - map.put(factory.getSupportedEncodingName(), factory); + map.put(Pair.newPair(factory.getSupportedEncodingName(), factory.getCurrentVersion()), factory); } catch (Exception ex) { logger.error("Failed to init dimension encoding factory " + clsName, ex); } @@ -95,9 +142,13 @@ public abstract class DimensionEncodingFactory { } } - /** Return the supported encoding name, corresponds to RowKeyColDesc.encodingName */ + /** + * Return the supported encoding name, corresponds to RowKeyColDesc.encodingName + */ abstract public String getSupportedEncodingName(); - /** Create a DimensionEncoding instance, with inputs corresponding to RowKeyColDesc.encodingName and RowKeyColDesc.encodingArgs */ + /** + * Create a DimensionEncoding instance, with inputs corresponding to RowKeyColDesc.encodingName and RowKeyColDesc.encodingArgs + */ abstract public DimensionEncoding createDimensionEncoding(String encodingName, String[] args); } http://git-wip-us.apache.org/repos/asf/kylin/blob/e1acc419/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java index 1e2d6dd..88e9533 100644 --- a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java +++ b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java @@ -57,6 +57,7 @@ public class TopNMeasureType extends MeasureType<TopNCounter<ByteArray>> { public static final String DATATYPE_TOPN = "topn"; public static final String CONFIG_ENCODING_PREFIX = "topn.encoding."; + public static final String CONFIG_ENCODING_VERSION_PREFIX = "topn.encoding_version."; public static final String CONFIG_AGG = "topn.aggregation"; public static final String CONFIG_ORDER = "topn.order"; @@ -418,11 +419,20 @@ public class TopNMeasureType extends MeasureType<TopNCounter<ByteArray>> { for (int i = 0; i < literalCols.size(); i++) { TblColRef colRef = literalCols.get(i); String encoding = function.getConfiguration().get(TopNMeasureType.CONFIG_ENCODING_PREFIX + colRef.getName()); + String encodingVersionStr = function.getConfiguration().get(TopNMeasureType.CONFIG_ENCODING_VERSION_PREFIX + colRef.getName()); if (StringUtils.isEmpty(encoding) || DictionaryDimEnc.ENCODING_NAME.equals(encoding)) { dimensionEncodings[i] = new DictionaryDimEnc(dictionaryMap.get(colRef)); } else { + int encodingVersion = 1; + if (!StringUtils.isEmpty(encodingVersionStr)) { + try { + encodingVersion = Integer.parseInt(encodingVersionStr); + } catch (NumberFormatException e) { + throw new RuntimeException(TopNMeasureType.CONFIG_ENCODING_VERSION_PREFIX + colRef.getName() + " has to be an integer"); + } + } Object[] encodingConf = DimensionEncoding.parseEncodingConf(encoding); - dimensionEncodings[i] = DimensionEncodingFactory.create((String) encodingConf[0], (String[]) encodingConf[1]); + dimensionEncodings[i] = DimensionEncodingFactory.create((String) encodingConf[0], (String[]) encodingConf[1], encodingVersion); } } http://git-wip-us.apache.org/repos/asf/kylin/blob/e1acc419/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java ---------------------------------------------------------------------- diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java index 64fde81..891248f 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java +++ b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java @@ -24,7 +24,6 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.UUID; import org.apache.commons.lang.StringUtils; @@ -76,7 +75,7 @@ import com.fasterxml.jackson.core.JsonParseException; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonMappingException; import com.google.common.base.Joiner; -import com.google.common.collect.Sets; +import com.google.common.collect.Maps; /** * CubeController is defined as Restful API entrance for UI. @@ -120,12 +119,13 @@ public class CubeController extends BasicController { @RequestMapping(value = "validEncodings", method = { RequestMethod.GET }) @ResponseBody - public Set<String> getValidEncodings() { - Set<String> encodings; + public Map<String, Integer> getValidEncodings() { + Map<String, Integer> encodings; try { encodings = DimensionEncodingFactory.getValidEncodings(); } catch (Exception e) { - return Sets.newTreeSet(); + logger.error("Error when getting valid encodings", e); + return Maps.newHashMap(); } return encodings; } @@ -360,7 +360,7 @@ public class CubeController extends BasicController { CubeDesc cubeDesc = cube.getDescriptor(); CubeDesc newCubeDesc = CubeDesc.getCopyOf(cubeDesc); - + KylinConfig config = cubeService.getConfig(); newCubeDesc.setName(newCubeName); newCubeDesc.setEngineType(config.getDefaultCubeEngine());