Repository: kylin Updated Branches: refs/heads/KYLIN-1122 ab7a76df5 -> 1e44c595b
KYLIN-1122 add UT and bug fix Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1e44c595 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1e44c595 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1e44c595 Branch: refs/heads/KYLIN-1122 Commit: 1e44c595bdb179368891ea0bd27750e9fd53e49b Parents: ab7a76d Author: Xiaoyu Wang <[email protected]> Authored: Tue Jan 12 18:49:42 2016 +0800 Committer: Xiaoyu Wang <[email protected]> Committed: Tue Jan 12 18:49:42 2016 +0800 ---------------------------------------------------------------------- .../kylin/measure/basic/BasicMeasureType.java | 4 --- .../kylin/measure/raw/RawMeasureType.java | 2 +- .../apache/kylin/measure/raw/RawSerializer.java | 29 ++++++++++++-------- .../kylin/metadata/model/FunctionDesc.java | 1 - 4 files changed, 19 insertions(+), 17 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/1e44c595/core-metadata/src/main/java/org/apache/kylin/measure/basic/BasicMeasureType.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/basic/BasicMeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/basic/BasicMeasureType.java index 9447ab0..b8d201e 100644 --- a/core-metadata/src/main/java/org/apache/kylin/measure/basic/BasicMeasureType.java +++ b/core-metadata/src/main/java/org/apache/kylin/measure/basic/BasicMeasureType.java @@ -77,10 +77,6 @@ public class BasicMeasureType extends MeasureType { if (rtype.isNumberFamily() == false) { throw new IllegalArgumentException("Return type for function " + funcName + " must be one of " + DataType.NUMBER_FAMILY); } - } else if (funcName.equals(FunctionDesc.FUNC_RAW)) { - if (!FunctionDesc.FUNC_RAW.equalsIgnoreCase(rtype.getName())) { - throw new IllegalArgumentException("Return type for function " + funcName + " must be raw "); - } } else { KylinConfig config = KylinConfig.getInstanceFromEnv(); if (config.isQueryIgnoreUnknownFunction() == false) http://git-wip-us.apache.org/repos/asf/kylin/blob/1e44c595/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java index 92044de..755d7f6 100644 --- a/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java +++ b/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java @@ -214,7 +214,7 @@ public class RawMeasureType extends MeasureType<List<ByteArray>> { ByteArray raw = rawIterator.next(); int key = BytesUtil.readUnsigned(raw.array(), 0, raw.array().length); String colValue = rawColDict.getValueFromId(key); - tuple.setMeasureValue(literalTupleIdx, colValue); + tuple.setDimensionValue(literalTupleIdx, colValue); } }; } http://git-wip-us.apache.org/repos/asf/kylin/blob/1e44c595/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawSerializer.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawSerializer.java b/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawSerializer.java index 884f777..25c70bc 100644 --- a/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawSerializer.java +++ b/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawSerializer.java @@ -30,6 +30,9 @@ import java.util.List; @SuppressWarnings("unused") public class RawSerializer extends DataTypeSerializer<List<ByteArray>> { + //one dictionary id value need 2 bytes, this buffer can contain ~ 512 * 1024 values + //FIXME to config this and RowConstants.ROWVALUE_BUFFER_SIZE in properties file + public static final int RAW_BUFFER_SIZE = 1024 * 1024;//1M public RawSerializer(DataType dataType) { } @@ -38,9 +41,9 @@ public class RawSerializer extends DataTypeSerializer<List<ByteArray>> { public int peekLength(ByteBuffer in) { int mark = in.position(); int len = 0; - if(in.hasRemaining()) { - int size = in.getInt(); - int bytes = in.getInt(); + if (in.hasRemaining()) { + int size = BytesUtil.readVInt(in); + int bytes = BytesUtil.readVInt(in); len = in.position() - mark + bytes; } in.position(mark); @@ -49,26 +52,30 @@ public class RawSerializer extends DataTypeSerializer<List<ByteArray>> { @Override public int maxLength() { - return 1024 * 1024; + return RAW_BUFFER_SIZE; } @Override public int getStorageBytesEstimate() { - return 1024 * 1024; + return RAW_BUFFER_SIZE; } @Override public void serialize(List<ByteArray> value, ByteBuffer out) { - if(value != null) { + if (value != null) { int bytes = 0; for (ByteArray array : value) { + //in BytesUtil.writeByteArray, writeVInt only need 1 byte, + //because dictionary id length is less than 127 + //so one value will need 2 bytes bytes += (array.length() + 1); } + //size and bytes value will need <=8 bytes + BytesUtil.writeVInt(value.size(), out); + BytesUtil.writeVInt(bytes, out); if (bytes > out.remaining()) { throw new RuntimeException("BufferOverflow! Please use one higher cardinality column for dimension column when build RAW cube!"); } - out.putInt(value.size()); - out.putInt(bytes); for (ByteArray array : value) { BytesUtil.writeByteArray(array.array(), out); } @@ -78,9 +85,9 @@ public class RawSerializer extends DataTypeSerializer<List<ByteArray>> { @Override public List<ByteArray> deserialize(ByteBuffer in) { List<ByteArray> value = null; - if(in.hasRemaining()) { - int size = in.getInt(); - int bytes = in.getInt(); + if (in.hasRemaining()) { + int size = BytesUtil.readVInt(in); + int bytes = BytesUtil.readVInt(in); value = new ArrayList<ByteArray>(size); for (int i = 0; i < size; i++) { value.add(new ByteArray(BytesUtil.readByteArray(in))); http://git-wip-us.apache.org/repos/asf/kylin/blob/1e44c595/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java index 624a73c..36c8722 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java @@ -40,7 +40,6 @@ public class FunctionDesc { public static final String FUNC_MIN = "MIN"; public static final String FUNC_MAX = "MAX"; public static final String FUNC_COUNT = "COUNT"; - public static final String FUNC_RAW = "RAW"; public static final String PARAMETER_TYPE_CONSTANT = "constant"; public static final String PARAMETER_TYPE_COLUMN = "column";
