Repository: kylin Updated Branches: refs/heads/1.4-rc 0456db93d -> 871154447
eBay-JIRA-524: automatical expansion when buffer overflow Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d02a5f5d Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d02a5f5d Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d02a5f5d Branch: refs/heads/1.4-rc Commit: d02a5f5d97078094dbbe7a0a809a7bcc81a3676c Parents: 0456db9 Author: kyotoYaho <nju_y...@apache.org> Authored: Tue May 17 21:10:58 2016 +0800 Committer: Zhong <yangzh...@lm-shc-16501214.corp.ebay.com> Committed: Fri May 27 10:15:18 2016 +0800 ---------------------------------------------------------------------- .../org/apache/kylin/dict/DictCodeSystem.java | 7 +++++ .../metadata/filter/ConstantTupleFilter.java | 30 ++++++++++++++++---- .../metadata/filter/TupleFilterSerializer.java | 24 ++++++++++++++-- 3 files changed, 54 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/d02a5f5d/core-dictionary/src/main/java/org/apache/kylin/dict/DictCodeSystem.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/DictCodeSystem.java b/core-dictionary/src/main/java/org/apache/kylin/dict/DictCodeSystem.java index 83ad20c..5a2044e 100644 --- a/core-dictionary/src/main/java/org/apache/kylin/dict/DictCodeSystem.java +++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictCodeSystem.java @@ -22,6 +22,8 @@ import java.nio.ByteBuffer; import org.apache.kylin.common.util.BytesUtil; import org.apache.kylin.metadata.filter.IFilterCodeSystem; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * A simple code system where all values are dictionary IDs (fixed length bytes) encoded as ISO-8859-1 strings. @@ -30,6 +32,8 @@ import org.apache.kylin.metadata.filter.IFilterCodeSystem; */ public class DictCodeSystem implements IFilterCodeSystem<String> { + private static final Logger logger = LoggerFactory.getLogger(DictCodeSystem.class); + public static final DictCodeSystem INSTANCE = new DictCodeSystem(); private DictCodeSystem() { @@ -57,6 +61,9 @@ public class DictCodeSystem implements IFilterCodeSystem<String> { //TODO: should use ISO-8859-1 rather than UTF8 @Override public void serialize(String value, ByteBuffer buffer) { + if(value.length() > 1000 ){ + logger.info("value length exceeds 1000: {}", value.substring(0,1000)); + } BytesUtil.writeUTFString(value, buffer); } http://git-wip-us.apache.org/repos/asf/kylin/blob/d02a5f5d/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ConstantTupleFilter.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ConstantTupleFilter.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ConstantTupleFilter.java index 074f163..a92c699 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ConstantTupleFilter.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ConstantTupleFilter.java @@ -18,6 +18,7 @@ package org.apache.kylin.metadata.filter; +import java.nio.BufferOverflowException; import java.nio.ByteBuffer; import java.util.Collection; import java.util.Collections; @@ -25,6 +26,8 @@ import java.util.HashSet; import org.apache.kylin.common.util.BytesUtil; import org.apache.kylin.metadata.tuple.IEvaluatableTuple; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * @@ -33,6 +36,8 @@ import org.apache.kylin.metadata.tuple.IEvaluatableTuple; */ public class ConstantTupleFilter extends TupleFilter { + private static final Logger logger = LoggerFactory.getLogger(ConstantTupleFilter.class); + public static final ConstantTupleFilter FALSE = new ConstantTupleFilter(); public static final ConstantTupleFilter TRUE = new ConstantTupleFilter((Object) null); // not sure of underlying code system, null is the only value that applies to all types @@ -89,11 +94,26 @@ public class ConstantTupleFilter extends TupleFilter { @SuppressWarnings({ "unchecked", "rawtypes" }) @Override public byte[] serialize(IFilterCodeSystem cs) { - ByteBuffer buffer = ByteBuffer.allocate(BUFFER_SIZE); - int size = this.constantValues.size(); - BytesUtil.writeVInt(size, buffer); - for (Object val : this.constantValues) { - cs.serialize(val, buffer); + ByteBuffer buffer; + int bufferSize = BUFFER_SIZE; + while(true){ + try{ + buffer = ByteBuffer.allocate(bufferSize); + int size = this.constantValues.size(); + BytesUtil.writeVInt(size, buffer); + logger.info("constantValues.size: {}", size); + for (Object val : this.constantValues) { + if(! (val instanceof String)){ + logger.error("Problematic constant value:" + val); + } + cs.serialize(val, buffer); + logger.info("Buffer Position: {}", buffer.position()); + } + break; + }catch (BufferOverflowException e) { + logger.info("Buffer size {} cannot hold the filter, resizing to 4 times", bufferSize); + bufferSize *= 4; + } } byte[] result = new byte[buffer.position()]; System.arraycopy(buffer.array(), 0, result, 0, buffer.position()); http://git-wip-us.apache.org/repos/asf/kylin/blob/d02a5f5d/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java index a394a51..628a97c 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java @@ -18,12 +18,15 @@ package org.apache.kylin.metadata.filter; +import java.nio.BufferOverflowException; import java.nio.ByteBuffer; import java.util.HashMap; import java.util.Map; import java.util.Stack; import org.apache.kylin.common.util.BytesUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * http://eli.thegreenplace.net/2011/09/29/an-interesting-tree-serialization-algorithm-from-dwarf @@ -33,11 +36,14 @@ import org.apache.kylin.common.util.BytesUtil; */ public class TupleFilterSerializer { + private static final Logger logger = LoggerFactory.getLogger(TupleFilterSerializer.class); + public interface Decorator { TupleFilter onSerialize(TupleFilter filter); } private static final int BUFFER_SIZE = 65536; + private static final int BUFFER_SIZE_MAX = 268435456; private static final Map<Integer, TupleFilter.FilterOperatorEnum> ID_OP_MAP = new HashMap<Integer, TupleFilter.FilterOperatorEnum>(); static { @@ -51,8 +57,22 @@ public class TupleFilterSerializer { } public static byte[] serialize(TupleFilter rootFilter, Decorator decorator, IFilterCodeSystem<?> cs) { - ByteBuffer buffer = ByteBuffer.allocate(BUFFER_SIZE); - internalSerialize(rootFilter, decorator, buffer, cs); + ByteBuffer buffer; + int bufferSize = BUFFER_SIZE; + while (true) { + try { + buffer = ByteBuffer.allocate(bufferSize); + internalSerialize(rootFilter, decorator, buffer, cs); + break; + } catch (BufferOverflowException e) { + logger.info("Buffer size {} cannot hold the filter, resizing to 4 times", bufferSize); + bufferSize *= 4; + if(bufferSize > BUFFER_SIZE_MAX){ + logger.error("Buffer size {} exceeds the limitation {}", bufferSize, BUFFER_SIZE_MAX); + throw e; + } + } + } byte[] result = new byte[buffer.position()]; System.arraycopy(buffer.array(), 0, result, 0, buffer.position()); return result;
