KYLIN-1478 support non-dic encoding Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9113a467 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9113a467 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9113a467
Branch: refs/heads/master Commit: 9113a4675430a0baffb8d133ba29465c5cd70b4a Parents: a490224 Author: shaofengshi <shaofeng...@apache.org> Authored: Mon Jun 6 17:37:11 2016 +0800 Committer: shaofengshi <shaofeng...@apache.org> Committed: Mon Jun 6 18:28:56 2016 +0800 ---------------------------------------------------------------------- .../apache/kylin/cube/model/RowKeyColDesc.java | 11 +- .../metadata/measure/TopNMeasureTypeTest.java | 62 ++++++ .../kylin/dimension/DimensionEncoding.java | 13 ++ .../apache/kylin/dimension/IntegerDimEnc.java | 16 ++ .../kylin/measure/topn/TopNMeasureType.java | 202 +++++++++++++------ .../kylin/metadata/model/FunctionDesc.java | 14 +- .../kylin/metadata/model/MeasureDesc.java | 2 + ...t_kylin_cube_without_slr_left_join_desc.json | 3 +- .../test_kylin_inner_join_model_desc.json | 17 +- .../test_kylin_left_join_model_desc.json | 16 +- .../test_case_data/sandbox/kylin_hive_conf.xml | 12 +- 11 files changed, 261 insertions(+), 107 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/9113a467/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java index 7e03355..e1be041 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java @@ -21,6 +21,7 @@ package org.apache.kylin.cube.model; import org.apache.commons.lang.StringUtils; import org.apache.kylin.common.util.StringUtil; import org.apache.kylin.dimension.DictionaryDimEnc; +import org.apache.kylin.dimension.DimensionEncoding; import org.apache.kylin.dimension.DimensionEncodingFactory; import org.apache.kylin.metadata.model.TblColRef; @@ -57,13 +58,9 @@ public class RowKeyColDesc { public void init() { Preconditions.checkState(StringUtils.isNotEmpty(this.encoding)); - String[] parts = this.encoding.split("\\s*[(),:]\\s*"); - if (parts == null || parts.length == 0 || parts[0].isEmpty()) - throw new IllegalArgumentException("Not supported row key col encoding: '" + this.encoding + "'"); - - this.encodingName = parts[0]; - this.encodingArgs = parts[parts.length - 1].isEmpty() // - ? StringUtil.subArray(parts, 1, parts.length - 1) : StringUtil.subArray(parts, 1, parts.length); + Object[] encodingConf = DimensionEncoding.parseEncodingConf(this.encoding); + this.encodingName = (String) encodingConf[0]; + this.encodingArgs = (String[])encodingConf[1]; if (!DimensionEncodingFactory.isVaildEncoding(this.encodingName)) throw new IllegalArgumentException("Not supported row key col encoding: '" + this.encoding + "'"); http://git-wip-us.apache.org/repos/asf/kylin/blob/9113a467/core-cube/src/test/java/org/apache/kylin/metadata/measure/TopNMeasureTypeTest.java ---------------------------------------------------------------------- diff --git a/core-cube/src/test/java/org/apache/kylin/metadata/measure/TopNMeasureTypeTest.java b/core-cube/src/test/java/org/apache/kylin/metadata/measure/TopNMeasureTypeTest.java new file mode 100644 index 0000000..3d3e7d0 --- /dev/null +++ b/core-cube/src/test/java/org/apache/kylin/metadata/measure/TopNMeasureTypeTest.java @@ -0,0 +1,62 @@ +package org.apache.kylin.metadata.measure; + +import static org.junit.Assert.*; + +import org.apache.kylin.common.util.LocalFileMetadataTestCase; +import org.apache.kylin.cube.CubeDescManager; +import org.apache.kylin.cube.model.CubeDesc; +import org.apache.kylin.measure.MeasureTypeFactory; +import org.apache.kylin.measure.topn.TopNMeasureType; +import org.apache.kylin.metadata.model.MeasureDesc; +import org.apache.kylin.metadata.model.TblColRef; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.util.List; + +/** + * Created by shishaofeng on 6/6/16. + */ +public class TopNMeasureTypeTest extends LocalFileMetadataTestCase { + + + @Before + public void setup() { + this.createTestMetadata(); + + } + + @After + public void clear() { + this.cleanupTestMetadata(); + } + + @Test + public void test() { + + CubeDesc desc = CubeDescManager.getInstance(this.getTestConfig()).getCubeDesc("test_kylin_cube_without_slr_left_join_desc"); + + MeasureDesc topSellerMeasure = null; + + for (MeasureDesc measureDesc : desc.getMeasures()) { + if (measureDesc.getName().equals("TOP_SELLER")) { + topSellerMeasure = measureDesc; + break; + } + } + TopNMeasureType measureType = (TopNMeasureType) MeasureTypeFactory.create(topSellerMeasure.getFunction().getExpression(), topSellerMeasure.getFunction().getReturnDataType()); + + + topSellerMeasure.getFunction().getConfiguration().clear(); + List<TblColRef> colsNeedDict = measureType.getColumnsNeedDictionary(topSellerMeasure.getFunction()); + + assertTrue(colsNeedDict != null && colsNeedDict.size() == 1); + + TblColRef sellerColRef = topSellerMeasure.getFunction().getParameter().getColRefs().get(1); + topSellerMeasure.getFunction().getConfiguration().put(TopNMeasureType.CONFIG_ENCODING_PREFIX + sellerColRef.getName(), "int:6"); + colsNeedDict = measureType.getColumnsNeedDictionary(topSellerMeasure.getFunction()); + + assertTrue(colsNeedDict.size() == 0); + } +} http://git-wip-us.apache.org/repos/asf/kylin/blob/9113a467/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncoding.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncoding.java b/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncoding.java index 5c32114..87dfc08 100644 --- a/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncoding.java +++ b/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncoding.java @@ -20,6 +20,7 @@ package org.apache.kylin.dimension; import java.io.Externalizable; +import org.apache.kylin.common.util.StringUtil; import org.apache.kylin.metadata.datatype.DataTypeSerializer; /** @@ -49,6 +50,18 @@ public abstract class DimensionEncoding implements Externalizable { return true; } + public static Object[] parseEncodingConf(String encoding) { + String[] parts = encoding.split("\\s*[(),:]\\s*"); + if (parts == null || parts.length == 0 || parts[0].isEmpty()) + throw new IllegalArgumentException("Not supported row key col encoding: '" + encoding + "'"); + + final String encodingName = parts[0]; + final String[] encodingArgs = parts[parts.length - 1].isEmpty() // + ? StringUtil.subArray(parts, 1, parts.length - 1) : StringUtil.subArray(parts, 1, parts.length); + + return new Object[] {encodingName, encodingArgs}; + } + /** return the fixed length of encoded bytes */ abstract public int getLengthOfEncoding(); http://git-wip-us.apache.org/repos/asf/kylin/blob/9113a467/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java b/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java index 9701447..3f01948 100644 --- a/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java +++ b/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java @@ -173,4 +173,20 @@ public class IntegerDimEnc extends DimensionEncoding { fixedLen = in.readShort(); } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + IntegerDimEnc that = (IntegerDimEnc) o; + + return fixedLen == that.fixedLen; + + } + + @Override + public int hashCode() { + return fixedLen; + } } http://git-wip-us.apache.org/repos/asf/kylin/blob/9113a467/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java index 009edef..0ff5b3a 100644 --- a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java +++ b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java @@ -18,24 +18,21 @@ package org.apache.kylin.measure.topn; -import java.util.Collection; -import java.util.Collections; -import java.util.Iterator; -import java.util.List; -import java.util.Map; - -import org.apache.kylin.common.util.ByteArray; -import org.apache.kylin.common.util.BytesUtil; +import java.util.*; + +import org.apache.commons.lang3.StringUtils; +import org.apache.kylin.common.util.*; import org.apache.kylin.common.util.Dictionary; +import org.apache.kylin.dimension.DictionaryDimEnc; +import org.apache.kylin.dimension.DimensionEncoding; +import org.apache.kylin.dimension.DimensionEncodingFactory; import org.apache.kylin.measure.MeasureAggregator; import org.apache.kylin.measure.MeasureIngester; import org.apache.kylin.measure.MeasureType; import org.apache.kylin.measure.MeasureTypeFactory; import org.apache.kylin.metadata.datatype.DataType; import org.apache.kylin.metadata.datatype.DataTypeSerializer; -import org.apache.kylin.metadata.model.FunctionDesc; -import org.apache.kylin.metadata.model.MeasureDesc; -import org.apache.kylin.metadata.model.TblColRef; +import org.apache.kylin.metadata.model.*; import org.apache.kylin.metadata.realization.CapabilityResult.CapabilityInfluence; import org.apache.kylin.metadata.realization.SQLDigest; import org.apache.kylin.metadata.tuple.Tuple; @@ -111,21 +108,38 @@ public class TopNMeasureType extends MeasureType<TopNCounter<ByteArray>> { @Override public MeasureIngester<TopNCounter<ByteArray>> newIngester() { return new MeasureIngester<TopNCounter<ByteArray>>() { + + private DimensionEncoding[] dimensionEncodings = null; + private List<TblColRef> literalCols = null; + private int keyLength = 0; + + private DimensionEncoding[] newDimensionEncodings = null; + private int newKeyLength = 0; + private boolean needReEncode = true; + @Override public TopNCounter<ByteArray> valueOf(String[] values, MeasureDesc measureDesc, Map<TblColRef, Dictionary<String>> dictionaryMap) { - if (values.length != 2) - throw new IllegalArgumentException(); - double counter = values[0] == null ? 0 : Double.parseDouble(values[0]); - String literal = values[1]; - // encode literal using dictionary - TblColRef literalCol = getTopNLiteralColumn(measureDesc.getFunction()); - Dictionary<String> dictionary = dictionaryMap.get(literalCol); - int keyEncodedValue = dictionary.getIdFromValue(literal); + if (dimensionEncodings == null) { + literalCols = getTopNLiteralColumn(measureDesc.getFunction()); + dimensionEncodings = getDimensionEncodings(measureDesc.getFunction(), literalCols, dictionaryMap); + for (DimensionEncoding encoding : dimensionEncodings) { + keyLength += encoding.getLengthOfEncoding(); + } - ByteArray key = new ByteArray(dictionary.getSizeOfId()); - BytesUtil.writeUnsigned(keyEncodedValue, key.array(), 0, dictionary.getSizeOfId()); + if (values.length != (literalCols.size() + 1)) { + throw new IllegalArgumentException(); + } + } + + final ByteArray key = new ByteArray(keyLength); + int offset = 0; + for (int i=0; i<dimensionEncodings.length; i++) { + byte[] valueBytes = Bytes.toBytes(values[i+1]); + dimensionEncodings[i].encode(valueBytes, valueBytes.length, key.array(), offset); + offset += dimensionEncodings[i].getLengthOfEncoding(); + } TopNCounter<ByteArray> topNCounter = new TopNCounter<ByteArray>(dataType.getPrecision() * TopNCounter.EXTRA_SPACE_RATE); topNCounter.offer(key, counter); @@ -136,30 +150,51 @@ public class TopNMeasureType extends MeasureType<TopNCounter<ByteArray>> { public TopNCounter<ByteArray> reEncodeDictionary(TopNCounter<ByteArray> value, MeasureDesc measureDesc, Map<TblColRef, Dictionary<String>> oldDicts, Map<TblColRef, Dictionary<String>> newDicts) { TopNCounter<ByteArray> topNCounter = value; - TblColRef colRef = getTopNLiteralColumn(measureDesc.getFunction()); - Dictionary<String> sourceDict = oldDicts.get(colRef); - Dictionary<String> mergedDict = newDicts.get(colRef); + if (newDimensionEncodings == null) { + literalCols = getTopNLiteralColumn(measureDesc.getFunction()); + dimensionEncodings = getDimensionEncodings(measureDesc.getFunction(), literalCols, oldDicts); + keyLength = 0; + boolean hasDictEncoding = false; + for (DimensionEncoding encoding : dimensionEncodings) { + keyLength += encoding.getLengthOfEncoding(); + if (encoding instanceof DictionaryDimEnc) { + hasDictEncoding = true; + } + } + + newDimensionEncodings = getDimensionEncodings(measureDesc.getFunction(), literalCols, newDicts); + newKeyLength = 0; + for (DimensionEncoding encoding : newDimensionEncodings) { + newKeyLength += encoding.getLengthOfEncoding(); + } + + needReEncode = hasDictEncoding; + } + + if (needReEncode == false) { + // no need re-encode + return topNCounter; + } int topNSize = topNCounter.size(); - byte[] newIdBuf = new byte[topNSize * mergedDict.getSizeOfId()]; - byte[] literal = new byte[sourceDict.getSizeOfValue()]; + byte[] newIdBuf = new byte[topNSize * newKeyLength]; int bufOffset = 0; for (Counter<ByteArray> c : topNCounter) { - int oldId = BytesUtil.readUnsigned(c.getItem().array(), c.getItem().offset(), c.getItem().length()); - int newId; - int size = sourceDict.getValueBytesFromId(oldId, literal, 0); - if (size < 0) { - newId = mergedDict.nullId(); - } else { - newId = mergedDict.getIdFromValueBytes(literal, 0, size); + int offset = c.getItem().offset(); + int innerBuffOffset = 0; + for (int i = 0; i < dimensionEncodings.length; i++) { + String dimValue = dimensionEncodings[i].decode(c.getItem().array(), offset, dimensionEncodings[i].getLengthOfEncoding()); + byte[] dimValueBytes = Bytes.toBytes(dimValue); + newDimensionEncodings[i].encode(dimValueBytes, dimValueBytes.length, newIdBuf, bufOffset + innerBuffOffset); + innerBuffOffset += newDimensionEncodings[i].getLengthOfEncoding(); + offset += dimensionEncodings[i].getLengthOfEncoding(); } - BytesUtil.writeUnsigned(newId, newIdBuf, bufOffset, mergedDict.getSizeOfId()); - c.getItem().set(newIdBuf, bufOffset, mergedDict.getSizeOfId()); - bufOffset += mergedDict.getSizeOfId(); + c.getItem().set(newIdBuf, bufOffset, newKeyLength); + bufOffset += newKeyLength; } - return value; + return topNCounter; } }; } @@ -171,8 +206,18 @@ public class TopNMeasureType extends MeasureType<TopNCounter<ByteArray>> { @Override public List<TblColRef> getColumnsNeedDictionary(FunctionDesc functionDesc) { - TblColRef literalCol = functionDesc.getParameter().getColRefs().get(1); - return Collections.singletonList(literalCol); + List <TblColRef> columnsNeedDict = Lists.newArrayList(); + List <TblColRef> allCols = functionDesc.getParameter().getColRefs(); + int start = (functionDesc.getParameter().isColumnType() == true) ? 1 : 0; + for (int i = start; i < allCols.size(); i++) { + TblColRef tblColRef = allCols.get(i); + String encoding = functionDesc.getConfiguration().get(CONFIG_ENCODING_PREFIX + tblColRef.getName()); + if(StringUtils.isEmpty(encoding) || DictionaryDimEnc.ENCODING_NAME.equals(encoding)) { + columnsNeedDict.add(tblColRef); + } + } + + return columnsNeedDict; } @Override @@ -189,13 +234,13 @@ public class TopNMeasureType extends MeasureType<TopNCounter<ByteArray>> { if (isTopNCompatibleSum(topN.getFunction(), onlyFunction) == false) return null; - TblColRef literalCol = getTopNLiteralColumn(topN.getFunction()); - if (unmatchedDimensions.contains(literalCol) == false) + List<TblColRef> literalCol = getTopNLiteralColumn(topN.getFunction()); + if (unmatchedDimensions.containsAll(literalCol) == false) return null; - if (digest.groupbyColumns.contains(literalCol) == false) + if (digest.groupbyColumns.containsAll(literalCol) == false) return null; - unmatchedDimensions.remove(literalCol); + unmatchedDimensions.removeAll(literalCol); unmatchedAggregations.remove(onlyFunction); return new CapabilityInfluence() { @Override @@ -212,11 +257,16 @@ public class TopNMeasureType extends MeasureType<TopNCounter<ByteArray>> { if (!isTopN(topN) || !sum.isSum()) return false; - if (sum.getParameter().getColRefs().isEmpty()) - return false; + TblColRef topnNumCol = getTopNNumericColumn(topN); + + if (sum.getParameter().getColRefs().isEmpty()) { + if (topnNumCol == null && sum.getParameter().getValue().equals(topN.getParameter().getValue())) + return true; + else + return false; + } TblColRef sumCol = sum.getParameter().getColRefs().get(0); - TblColRef topnNumCol = getTopNNumericColumn(topN); return sumCol.equals(topnNumCol); } @@ -233,9 +283,9 @@ public class TopNMeasureType extends MeasureType<TopNCounter<ByteArray>> { @Override public void adjustSqlDigest(MeasureDesc measureDesc, SQLDigest sqlDigest) { FunctionDesc topnFunc = measureDesc.getFunction(); - TblColRef topnLiteralCol = getTopNLiteralColumn(topnFunc); + List<TblColRef> topnLiteralCol = getTopNLiteralColumn(topnFunc); - if (sqlDigest.groupbyColumns.contains(topnLiteralCol) == false) + if (sqlDigest.groupbyColumns.containsAll(topnLiteralCol) == false) return; if (sqlDigest.aggregations.size() > 1) { @@ -251,8 +301,8 @@ public class TopNMeasureType extends MeasureType<TopNCounter<ByteArray>> { } sqlDigest.aggregations = Lists.newArrayList(topnFunc); - sqlDigest.groupbyColumns.remove(topnLiteralCol); - sqlDigest.metricColumns.add(topnLiteralCol); + sqlDigest.groupbyColumns.removeAll(topnLiteralCol); + sqlDigest.metricColumns.addAll(topnLiteralCol); } @Override @@ -267,12 +317,17 @@ public class TopNMeasureType extends MeasureType<TopNCounter<ByteArray>> { @Override public IAdvMeasureFiller getAdvancedTupleFiller(FunctionDesc function, TupleInfo tupleInfo, Map<TblColRef, Dictionary<String>> dictionaryMap) { - final TblColRef literalCol = getTopNLiteralColumn(function); + final List<TblColRef> literalCols = getTopNLiteralColumn(function); final TblColRef numericCol = getTopNNumericColumn(function); - final Dictionary<String> topNColDict = dictionaryMap.get(literalCol); - final int literalTupleIdx = tupleInfo.hasColumn(literalCol) ? tupleInfo.getColumnIndex(literalCol) : -1; + final int[] literalTupleIdx = new int[literalCols.size()]; + final DimensionEncoding[] dimensionEncodings = getDimensionEncodings(function, literalCols, dictionaryMap); + for (int i=0; i< literalCols.size(); i++) { + TblColRef colRef = literalCols.get(i); + literalTupleIdx[i] = tupleInfo.hasColumn(colRef) ? tupleInfo.getColumnIndex(colRef) : -1; + } + // for TopN, the aggr must be SUM, so the number fill into the column position (without rewrite) - final int numericTupleIdx = tupleInfo.hasColumn(numericCol) ? tupleInfo.getColumnIndex(numericCol) : -1; + final int numericTupleIdx = (numericCol != null && tupleInfo.hasColumn(numericCol)) ? tupleInfo.getColumnIndex(numericCol) : -1; return new IAdvMeasureFiller() { private TopNCounter<ByteArray> topNCounter; private Iterator<Counter<ByteArray>> topNCounterIterator; @@ -297,23 +352,50 @@ public class TopNMeasureType extends MeasureType<TopNCounter<ByteArray>> { throw new IllegalStateException(); Counter<ByteArray> counter = topNCounterIterator.next(); - int key = BytesUtil.readUnsigned(counter.getItem().array(), counter.getItem().offset(), counter.getItem().length()); - String colValue = topNColDict.getValueFromId(key); - tuple.setDimensionValue(literalTupleIdx, colValue); + int offset = counter.getItem().offset(); + for (int i = 0; i< dimensionEncodings.length; i++) { + String colValue = dimensionEncodings[i].decode(counter.getItem().array(), offset, dimensionEncodings[i].getLengthOfEncoding()); + tuple.setDimensionValue(literalTupleIdx[i], colValue); + offset += dimensionEncodings[i].getLengthOfEncoding(); + } tuple.setMeasureValue(numericTupleIdx, counter.getCount()); } }; } + private static DimensionEncoding[] getDimensionEncodings(FunctionDesc function, List<TblColRef> literalCols, Map<TblColRef, Dictionary<String>> dictionaryMap) { + final DimensionEncoding[] dimensionEncodings = new DimensionEncoding[literalCols.size()]; + for (int i=0; i< literalCols.size(); i++) { + TblColRef colRef = literalCols.get(i); + String encoding = function.getConfiguration().get(TopNMeasureType.CONFIG_ENCODING_PREFIX + colRef.getName()); + if(StringUtils.isEmpty(encoding) || DictionaryDimEnc.ENCODING_NAME.equals(encoding)) { + dimensionEncodings[i] = new DictionaryDimEnc(dictionaryMap.get(colRef)); + } else { + Object[] encodingConf = DimensionEncoding.parseEncodingConf(encoding); + dimensionEncodings[i] = DimensionEncodingFactory.create((String) encodingConf[0], (String[]) encodingConf[1]); + } + } + + return dimensionEncodings; + } + private TblColRef getTopNNumericColumn(FunctionDesc functionDesc) { - return functionDesc.getParameter().getColRefs().get(0); + if (functionDesc.getParameter().isColumnType() == true) { + return functionDesc.getParameter().getColRefs().get(0); + } + return null; } - private TblColRef getTopNLiteralColumn(FunctionDesc functionDesc) { - return functionDesc.getParameter().getColRefs().get(1); + private List<TblColRef> getTopNLiteralColumn(FunctionDesc functionDesc) { + List<TblColRef> allColumns = functionDesc.getParameter().getColRefs(); + if (functionDesc.getParameter().isColumnType() == false) { + return allColumns; + } + return allColumns.subList(1, allColumns.size()); } private boolean isTopN(FunctionDesc functionDesc) { return FUNC_TOP_N.equalsIgnoreCase(functionDesc.getExpression()); } + } http://git-wip-us.apache.org/repos/asf/kylin/blob/9113a467/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java index 0906d5f..9450555 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java @@ -20,6 +20,7 @@ package org.apache.kylin.metadata.model; import java.util.*; +import com.fasterxml.jackson.annotation.JsonInclude; import org.apache.kylin.measure.MeasureType; import org.apache.kylin.measure.MeasureTypeFactory; import org.apache.kylin.measure.basic.BasicMeasureType; @@ -61,8 +62,9 @@ public class FunctionDesc { @JsonProperty("returntype") private String returnType; - @JsonProperty("configurations") - private HashMap<String, String> configurations = new LinkedHashMap<String, String>(); + @JsonProperty("configuration") + @JsonInclude(JsonInclude.Include.NON_NULL) + private HashMap<String, String> configuration = new LinkedHashMap<String, String>(); private DataType returnDataType; private MeasureType<?> measureType; @@ -265,12 +267,12 @@ public class FunctionDesc { return null; } - public HashMap<String, String> getConfigurations() { - return configurations; + public HashMap<String, String> getConfiguration() { + return configuration; } - public void setConfigurations(HashMap<String, String> configurations) { - this.configurations = configurations; + public void setConfiguration(HashMap<String, String> configurations) { + this.configuration = configurations; } http://git-wip-us.apache.org/repos/asf/kylin/blob/9113a467/core-metadata/src/main/java/org/apache/kylin/metadata/model/MeasureDesc.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/MeasureDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/MeasureDesc.java index c8ac1a9..dde791c 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/MeasureDesc.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/MeasureDesc.java @@ -20,6 +20,7 @@ package org.apache.kylin.metadata.model; import com.fasterxml.jackson.annotation.JsonAutoDetect; import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; +import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; /** @@ -34,6 +35,7 @@ public class MeasureDesc { @JsonProperty("function") private FunctionDesc function; @JsonProperty("dependent_measure_ref") + @JsonInclude(JsonInclude.Include.NON_NULL) private String dependentMeasureRef; public String getName() { http://git-wip-us.apache.org/repos/asf/kylin/blob/9113a467/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json index a501bd1..ca1b35c 100644 --- a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json +++ b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json @@ -156,7 +156,8 @@ "next_parameter" : null } }, - "returntype" : "topn(100)" + "returntype" : "topn(100)", + "configuration": {"topn.encoding.SELLER_ID" : "int:4"} }, "dependent_measure_ref" : null }, { http://git-wip-us.apache.org/repos/asf/kylin/blob/9113a467/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_model_desc.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_model_desc.json b/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_model_desc.json index a8fe363..f5333b0 100644 --- a/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_model_desc.json +++ b/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_model_desc.json @@ -38,7 +38,10 @@ "lstg_format_name", "LSTG_SITE_ID", "SLR_SEGMENT_CD", - "seller_id" + "TRANS_ID", + "CAL_DT", + "LEAF_CATEG_ID", + "SELLER_ID" ] }, { @@ -76,20 +79,12 @@ "cal_dt", "week_beg_dt" ] - }, - { - "table": "default.test_kylin_fact", - "columns": [ - "TRANS_ID", - "CAL_DT", - "LEAF_CATEG_ID", - "SELLER_ID" - ] } ], "metrics": [ "PRICE", - "ITEM_COUNT" + "ITEM_COUNT", + "SELLER_ID" ], "last_modified" : 1422435345352, "fact_table" : "DEFAULT.TEST_KYLIN_FACT", http://git-wip-us.apache.org/repos/asf/kylin/blob/9113a467/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_model_desc.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_model_desc.json b/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_model_desc.json index c85e5d2..8be6dc5 100644 --- a/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_model_desc.json +++ b/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_model_desc.json @@ -54,10 +54,12 @@ { "table": "default.test_kylin_fact", "columns": [ + "TRANS_ID", + "CAL_DT", "lstg_format_name", "LSTG_SITE_ID", "SLR_SEGMENT_CD", - "seller_id" + "SELLER_ID" ] }, { @@ -95,20 +97,12 @@ "cal_dt", "week_beg_dt" ] - }, - { - "table": "default.test_kylin_fact", - "columns": [ - "TRANS_ID", - "CAL_DT", - "LEAF_CATEG_ID", - "SELLER_ID" - ] } ], "metrics": [ "PRICE", - "ITEM_COUNT" + "ITEM_COUNT", + "SELLER_ID" ], "last_modified": 1422435345352, "fact_table": "DEFAULT.TEST_KYLIN_FACT", http://git-wip-us.apache.org/repos/asf/kylin/blob/9113a467/examples/test_case_data/sandbox/kylin_hive_conf.xml ---------------------------------------------------------------------- diff --git a/examples/test_case_data/sandbox/kylin_hive_conf.xml b/examples/test_case_data/sandbox/kylin_hive_conf.xml index f7ae507..e75f191 100644 --- a/examples/test_case_data/sandbox/kylin_hive_conf.xml +++ b/examples/test_case_data/sandbox/kylin_hive_conf.xml @@ -54,18 +54,8 @@ --> <property> - <name>hive.merge.mapfiles</name> - <value>true</value> - <description>Enable hive file merge on mapper only job</description> - </property> - <property> - <name>hive.merge.mapredfiles</name> - <value>true</value> - <description>Enable hive file merge on map-reduce job</description> - </property> - <property> <name>hive.merge.size.per.task</name> - <value>64000000</value> + <value>32000000</value> <description>Size for the merged file</description> </property> </configuration> \ No newline at end of file