This is an automated email from the ASF dual-hosted git repository. billyliu pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/master by this push: new 8350de4 KYLIN-3293, fix FixedLenHexDimEnc that return a wrong code length leads to cut bytes error. 8350de4 is described below commit 8350de4493ac792a62878a1aebf6588b3119e4bb Author: Jiatao Tao <245915...@qq.com> AuthorDate: Fri Mar 16 22:15:51 2018 +0800 KYLIN-3293, fix FixedLenHexDimEnc that return a wrong code length leads to cut bytes error. --- .../cube/gridtable/TrimmedCubeCodeSystem.java | 4 +-- .../kylin/gridtable/TrimmedCubeCodeSystemTest.java | 41 ++++++++++++++++++++++ .../apache/kylin/dimension/FixedLenHexDimEnc.java | 6 ++-- 3 files changed, 47 insertions(+), 4 deletions(-) diff --git a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/TrimmedCubeCodeSystem.java b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/TrimmedCubeCodeSystem.java index a0b230e..261e501 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/TrimmedCubeCodeSystem.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/TrimmedCubeCodeSystem.java @@ -51,7 +51,7 @@ public class TrimmedCubeCodeSystem extends CubeCodeSystem { serializer.serialize(value, buf); } - private static void writeDimensionEncoding(DimensionEncoding encoding, ByteBuffer out) { + public static void writeDimensionEncoding(DimensionEncoding encoding, ByteBuffer out) { try { if (encoding == null) { BytesUtil.writeVInt(1, out); @@ -71,7 +71,7 @@ public class TrimmedCubeCodeSystem extends CubeCodeSystem { } } - private static DimensionEncoding readDimensionEncoding(ByteBuffer in) { + public static DimensionEncoding readDimensionEncoding(ByteBuffer in) { try { int isNull = BytesUtil.readVInt(in); if (isNull == 1) { diff --git a/core-cube/src/test/java/org/apache/kylin/gridtable/TrimmedCubeCodeSystemTest.java b/core-cube/src/test/java/org/apache/kylin/gridtable/TrimmedCubeCodeSystemTest.java new file mode 100644 index 0000000..dc3c762 --- /dev/null +++ b/core-cube/src/test/java/org/apache/kylin/gridtable/TrimmedCubeCodeSystemTest.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kylin.gridtable; + +import static org.apache.kylin.cube.gridtable.TrimmedCubeCodeSystem.readDimensionEncoding; +import static org.apache.kylin.cube.gridtable.TrimmedCubeCodeSystem.writeDimensionEncoding; + +import java.nio.ByteBuffer; + +import org.apache.kylin.dimension.DimensionEncoding; +import org.apache.kylin.dimension.FixedLenHexDimEnc; +import org.junit.Assert; +import org.junit.Test; + +public class TrimmedCubeCodeSystemTest { + @Test + public void testFixLenHexEncSerDser() { + FixedLenHexDimEnc enc = new FixedLenHexDimEnc(6); + ByteBuffer buff = ByteBuffer.allocate(1024); + writeDimensionEncoding(enc, buff); + buff.flip(); + DimensionEncoding dimensionEncoding = readDimensionEncoding(buff); + Assert.assertEquals(3, dimensionEncoding.asDataTypeSerializer().peekLength(null)); + } +} diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/FixedLenHexDimEnc.java b/core-metadata/src/main/java/org/apache/kylin/dimension/FixedLenHexDimEnc.java index a931450..1d7e3c9 100644 --- a/core-metadata/src/main/java/org/apache/kylin/dimension/FixedLenHexDimEnc.java +++ b/core-metadata/src/main/java/org/apache/kylin/dimension/FixedLenHexDimEnc.java @@ -40,7 +40,7 @@ import com.google.common.base.Preconditions; * <p> * 1. "FFFF" will become null encode and decode * 2. "AB" will become "AB00" - * + * * <p> * Due to these limitations hex representation of hash values(with no padding, better with even characters) is more suitable */ @@ -166,7 +166,7 @@ public class FixedLenHexDimEnc extends DimensionEncoding implements Serializable byte[] value = Bytes.toBytes(valueStr); int valueLen = value.length; int endOffset = outputOffset + bytelen; - + if (valueLen > hexLength) { if (avoidVerbose++ % 10000 == 0) { logger.warn("Expect at most " + hexLength + " bytes, but got " + valueLen + ", will truncate, value string: " + Bytes.toString(value, 0, valueLen) + " times:" + avoidVerbose); @@ -274,11 +274,13 @@ public class FixedLenHexDimEnc extends DimensionEncoding implements Serializable @Override public void writeExternal(ObjectOutput out) throws IOException { out.writeShort(hexLength); + out.writeShort(bytelen); } @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException { hexLength = in.readShort(); + bytelen = in.readShort(); } private boolean isF(byte[] value, int offset, int length) { -- To stop receiving notification emails like this one, please contact billy...@apache.org.