Repository: kylin Updated Branches: refs/heads/yang21 d4715b559 -> 150bab61a
revert KYLIN-2199 because KYLIN-2191 is not a major fix Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/150bab61 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/150bab61 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/150bab61 Branch: refs/heads/yang21 Commit: 150bab61a77c0900bf8d43f569b710d8b9177f0d Parents: d4715b5 Author: Hongbin Ma <mahong...@apache.org> Authored: Tue Nov 22 15:32:31 2016 +0800 Committer: Hongbin Ma <mahong...@apache.org> Committed: Tue Nov 22 15:51:08 2016 +0800 ---------------------------------------------------------------------- .../gridtable/DimEncodingPreserveOrderTest.java | 4 +- .../dimension/DimensionEncodingFactory.java | 4 - .../apache/kylin/dimension/IntegerDimEnc.java | 3 +- .../apache/kylin/dimension/IntegerDimEncV2.java | 228 ------------------- .../kylin/dimension/IntegerDimEncTest.java | 16 +- 5 files changed, 11 insertions(+), 244 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/150bab61/core-cube/src/test/java/org/apache/kylin/gridtable/DimEncodingPreserveOrderTest.java ---------------------------------------------------------------------- diff --git a/core-cube/src/test/java/org/apache/kylin/gridtable/DimEncodingPreserveOrderTest.java b/core-cube/src/test/java/org/apache/kylin/gridtable/DimEncodingPreserveOrderTest.java index d572e56..87d37be 100644 --- a/core-cube/src/test/java/org/apache/kylin/gridtable/DimEncodingPreserveOrderTest.java +++ b/core-cube/src/test/java/org/apache/kylin/gridtable/DimEncodingPreserveOrderTest.java @@ -27,7 +27,7 @@ import org.apache.kylin.common.util.ByteArray; import org.apache.kylin.common.util.Bytes; import org.apache.kylin.dimension.DimensionEncoding; import org.apache.kylin.dimension.FixedLenHexDimEnc; -import org.apache.kylin.dimension.IntegerDimEncV2; +import org.apache.kylin.dimension.IntegerDimEnc; import org.apache.kylin.dimension.OneMoreByteVLongDimEnc; import org.junit.BeforeClass; import org.junit.Test; @@ -99,7 +99,7 @@ public class DimEncodingPreserveOrderTest { @Test public void testVLongDimEncPreserveOrder() { for (int i = 1; i <= successValue.size(); i++) { - IntegerDimEncV2 enc = new IntegerDimEncV2(i); + IntegerDimEnc enc = new IntegerDimEnc(i); List<ByteArray> encodedValues = Lists.newArrayList(); for (long value : successValue.get(i - 1)) { encodedValues.add(encode(enc, value)); http://git-wip-us.apache.org/repos/asf/kylin/blob/150bab61/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncodingFactory.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncodingFactory.java b/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncodingFactory.java index b617a19..2b6f894 100644 --- a/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncodingFactory.java +++ b/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncodingFactory.java @@ -117,10 +117,6 @@ public abstract class DimensionEncodingFactory { map.put(Pair.newPair(IntegerDimEnc.ENCODING_NAME, value.getCurrentVersion()), value); } { - IntegerDimEncV2.Factory value = new IntegerDimEncV2.Factory(); - map.put(Pair.newPair(IntegerDimEncV2.ENCODING_NAME, value.getCurrentVersion()), value); - } - { FixedLenHexDimEnc.Factory value = new FixedLenHexDimEnc.Factory(); map.put(Pair.newPair(FixedLenHexDimEnc.ENCODING_NAME, value.getCurrentVersion()), value); } http://git-wip-us.apache.org/repos/asf/kylin/blob/150bab61/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java b/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java index 983af9a..bcd6283 100644 --- a/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java +++ b/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java @@ -37,7 +37,6 @@ import org.slf4j.LoggerFactory; * -2^(8*N-1) is not supported because the slot is reserved for null values. * -2^(8*N-1) will be encoded with warn, and its output will be null */ -@Deprecated//due to a fatal bug (KYLIN-2191) public class IntegerDimEnc extends DimensionEncoding { private static final long serialVersionUID = 1L; @@ -132,7 +131,7 @@ public class IntegerDimEnc extends DimensionEncoding { //only take useful bytes integer = integer & MASK[fixedLen]; - boolean positive = (integer & ((0x80) << ((fixedLen - 1) << 3))) == 0; + boolean positive = (integer & ((0x80L) << ((fixedLen - 1) << 3))) == 0; if (!positive) { integer |= (~MASK[fixedLen]); } http://git-wip-us.apache.org/repos/asf/kylin/blob/150bab61/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEncV2.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEncV2.java b/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEncV2.java deleted file mode 100644 index 1a54664..0000000 --- a/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEncV2.java +++ /dev/null @@ -1,228 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.dimension; - -import java.io.IOException; -import java.io.ObjectInput; -import java.io.ObjectOutput; -import java.nio.ByteBuffer; -import java.util.Arrays; - -import org.apache.kylin.common.util.Bytes; -import org.apache.kylin.common.util.BytesUtil; -import org.apache.kylin.metadata.datatype.DataTypeSerializer; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * replacement for IntDimEnc, the diff is IntegerDimEnc supports negative values - * for IntegerDimEnc(N), the supported range is (-2^(8*N-1),2^(8*N-1)) - * - * -2^(8*N-1) is not supported because the slot is reserved for null values. - * -2^(8*N-1) will be encoded with warn, and its output will be null - */ -public class IntegerDimEncV2 extends DimensionEncoding { - private static final long serialVersionUID = 1L; - - private static Logger logger = LoggerFactory.getLogger(IntegerDimEncV2.class); - - private static final long[] CAP = { 0, 0x7fL, 0x7fffL, 0x7fffffL, 0x7fffffffL, 0x7fffffffffL, 0x7fffffffffffL, 0x7fffffffffffffL, 0x7fffffffffffffffL }; - private static final long[] MASK = { 0, 0xffL, 0xffffL, 0xffffffL, 0xffffffffL, 0xffffffffffL, 0xffffffffffffL, 0xffffffffffffffL, 0xffffffffffffffffL }; - private static final long[] TAIL = { 0, 0x80L, 0x8000L, 0x800000L, 0x80000000L, 0x8000000000L, 0x800000000000L, 0x80000000000000L, 0x8000000000000000L }; - static { - for (int i = 1; i < TAIL.length; ++i) { - long head = ~MASK[i]; - TAIL[i] = head | TAIL[i]; - } - } - - public static final String ENCODING_NAME = "integer"; - - public static class Factory extends DimensionEncodingFactory { - @Override - public String getSupportedEncodingName() { - return ENCODING_NAME; - } - - @Override - protected int getCurrentVersion() { - return 2; - } - - @Override - public DimensionEncoding createDimensionEncoding(String encodingName, String[] args) { - return new IntegerDimEncV2(Integer.parseInt(args[0])); - } - }; - - // ============================================================================ - - private int fixedLen; - - transient private int avoidVerbose = 0; - transient private int avoidVerbose2 = 0; - - //no-arg constructor is required for Externalizable - public IntegerDimEncV2() { - } - - public IntegerDimEncV2(int len) { - if (len <= 0 || len >= CAP.length) - throw new IllegalArgumentException(); - - this.fixedLen = len; - } - - @Override - public int getLengthOfEncoding() { - return fixedLen; - } - - @Override - public void encode(byte[] value, int valueLen, byte[] output, int outputOffset) { - if (value == null) { - Arrays.fill(output, outputOffset, outputOffset + fixedLen, NULL); - return; - } - - encode(Bytes.toString(value, 0, valueLen), output, outputOffset); - } - - void encode(String valueStr, byte[] output, int outputOffset) { - if (valueStr == null) { - Arrays.fill(output, outputOffset, outputOffset + fixedLen, NULL); - return; - } - - long integer = Long.parseLong(valueStr); - if (integer > CAP[fixedLen] || integer < TAIL[fixedLen]) { - if (avoidVerbose++ % 10000 == 0) { - logger.warn("Expect at most " + fixedLen + " bytes, but got " + valueStr + ", will truncate, hit times:" + avoidVerbose); - } - } - - if (integer == TAIL[fixedLen]) { - if (avoidVerbose2++ % 10000 == 0) { - logger.warn("Value " + valueStr + " does not fit into " + fixedLen + " bytes "); - } - } - - BytesUtil.writeLong(integer + CAP[fixedLen], output, outputOffset, fixedLen);//apply an offset to preserve binary order, overflow is okay - } - - @Override - public String decode(byte[] bytes, int offset, int len) { - if (isNull(bytes, offset, len)) { - return null; - } - - long integer = BytesUtil.readLong(bytes, offset, len) - CAP[fixedLen]; - - //only take useful bytes - integer = integer & MASK[fixedLen]; - boolean positive = (integer & ((0x80L) << ((fixedLen - 1) << 3))) == 0; - if (!positive) { - integer |= (~MASK[fixedLen]); - } - - return String.valueOf(integer); - } - - @Override - public DataTypeSerializer<Object> asDataTypeSerializer() { - return new IntegerSerializer(); - } - - public class IntegerSerializer extends DataTypeSerializer<Object> { - // be thread-safe and avoid repeated obj creation - private ThreadLocal<byte[]> current = new ThreadLocal<byte[]>(); - - private byte[] currentBuf() { - byte[] buf = current.get(); - if (buf == null) { - buf = new byte[fixedLen]; - current.set(buf); - } - return buf; - } - - @Override - public void serialize(Object value, ByteBuffer out) { - byte[] buf = currentBuf(); - String valueStr = value == null ? null : value.toString(); - encode(valueStr, buf, 0); - out.put(buf); - } - - @Override - public Object deserialize(ByteBuffer in) { - byte[] buf = currentBuf(); - in.get(buf); - return decode(buf, 0, buf.length); - } - - @Override - public int peekLength(ByteBuffer in) { - return fixedLen; - } - - @Override - public int maxLength() { - return fixedLen; - } - - @Override - public int getStorageBytesEstimate() { - return fixedLen; - } - - @Override - public Object valueOf(String str) { - return str; - } - } - - @Override - public void writeExternal(ObjectOutput out) throws IOException { - out.writeShort(fixedLen); - } - - @Override - public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException { - fixedLen = in.readShort(); - } - - @Override - public boolean equals(Object o) { - if (this == o) - return true; - if (o == null || getClass() != o.getClass()) - return false; - - IntegerDimEncV2 that = (IntegerDimEncV2) o; - - return fixedLen == that.fixedLen; - - } - - @Override - public int hashCode() { - return fixedLen; - } -} http://git-wip-us.apache.org/repos/asf/kylin/blob/150bab61/core-metadata/src/test/java/org/apache/kylin/dimension/IntegerDimEncTest.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/test/java/org/apache/kylin/dimension/IntegerDimEncTest.java b/core-metadata/src/test/java/org/apache/kylin/dimension/IntegerDimEncTest.java index 9924053..0110614 100644 --- a/core-metadata/src/test/java/org/apache/kylin/dimension/IntegerDimEncTest.java +++ b/core-metadata/src/test/java/org/apache/kylin/dimension/IntegerDimEncTest.java @@ -76,24 +76,24 @@ public class IntegerDimEncTest { @Test public void testConstructor() { try { - new IntegerDimEncV2(0); + new IntegerDimEnc(0); Assert.fail(); } catch (IllegalArgumentException e) { // expect } try { - new IntegerDimEncV2(9); + new IntegerDimEnc(9); Assert.fail(); } catch (IllegalArgumentException e) { // expect } - new IntegerDimEncV2(8); + new IntegerDimEnc(8); } @Test public void testNull() { for (int i = 1; i < 9; i++) { - IntegerDimEncV2 enc = new IntegerDimEncV2(i); + IntegerDimEnc enc = new IntegerDimEnc(i); byte[] buf = new byte[enc.getLengthOfEncoding()]; enc.encode(null, 0, buf, 0); @@ -113,7 +113,7 @@ public class IntegerDimEncTest { @Test public void testEncodeDecode() { for (int i = 1; i <= successValue.size(); i++) { - IntegerDimEncV2 enc = new IntegerDimEncV2(i); + IntegerDimEnc enc = new IntegerDimEnc(i); for (long value : successValue.get(i - 1)) { testEncodeDecode(enc, value); } @@ -129,7 +129,7 @@ public class IntegerDimEncTest { } } - private void testEncodeDecode(IntegerDimEncV2 enc, long value) { + private void testEncodeDecode(IntegerDimEnc enc, long value) { String valueStr = "" + value; byte[] buf = new byte[enc.getLengthOfEncoding()]; byte[] bytes = Bytes.toBytes(valueStr); @@ -141,7 +141,7 @@ public class IntegerDimEncTest { @Test public void testSerDes() { for (int i = 1; i <= successValue.size(); i++) { - IntegerDimEncV2 enc = new IntegerDimEncV2(i); + IntegerDimEnc enc = new IntegerDimEnc(i); testSerDes(enc, 127); for (long value : successValue.get(i - 1)) { @@ -158,7 +158,7 @@ public class IntegerDimEncTest { } } - private void testSerDes(IntegerDimEncV2 enc, long value) { + private void testSerDes(IntegerDimEnc enc, long value) { DataTypeSerializer<Object> ser = enc.asDataTypeSerializer(); byte[] buf = new byte[enc.getLengthOfEncoding()]; String valueStr = "" + value;
