KYLIN-2662 report error when writing super large dict cell
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/73bd0339 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/73bd0339 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/73bd0339 Branch: refs/heads/master Commit: 73bd0339fea74944e465921ed4283215b01e2c67 Parents: 247f3cd Author: Roger Shi <[email protected]> Authored: Tue Jun 20 16:32:01 2017 +0800 Committer: Hongbin Ma <[email protected]> Committed: Tue Jun 20 17:16:03 2017 +0800 ---------------------------------------------------------------------- .../main/java/org/apache/kylin/common/util/BytesUtil.java | 4 ++++ .../src/main/java/org/apache/kylin/dict/TrieDictionary.java | 5 ++++- .../java/org/apache/kylin/dict/TrieDictionaryBuilder.java | 8 ++++++++ 3 files changed, 16 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/73bd0339/core-common/src/main/java/org/apache/kylin/common/util/BytesUtil.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/util/BytesUtil.java b/core-common/src/main/java/org/apache/kylin/common/util/BytesUtil.java index 49f1ee0..8cc559e 100644 --- a/core-common/src/main/java/org/apache/kylin/common/util/BytesUtil.java +++ b/core-common/src/main/java/org/apache/kylin/common/util/BytesUtil.java @@ -177,6 +177,10 @@ public class BytesUtil { return r; } + public static boolean isPositiveShort(int i) { + return (i & 0xFFFF7000) == 0; + } + // from WritableUtils // ============================================================================ http://git-wip-us.apache.org/repos/asf/kylin/blob/73bd0339/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionary.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionary.java b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionary.java index 8849015..53daeb7 100644 --- a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionary.java +++ b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionary.java @@ -29,7 +29,6 @@ import java.io.ObjectOutputStream; import java.io.PrintStream; import java.util.Arrays; - import org.apache.kylin.common.util.Bytes; import org.apache.kylin.common.util.BytesUtil; import org.apache.kylin.common.util.ClassUtil; @@ -101,6 +100,10 @@ public class TrieDictionary<T> extends CacheDictionary<T> { this.sizeNoValuesBeneath = headIn.read(); this.baseId = headIn.readShort(); this.maxValueLength = headIn.readShort(); + if (maxValueLength < 0) { + throw new IllegalStateException("maxValueLength is negative (" + maxValueLength + + "). Dict value is too long, whose length is larger than " + Short.MAX_VALUE); + } String converterName = headIn.readUTF(); if (converterName.isEmpty() == false) http://git-wip-us.apache.org/repos/asf/kylin/blob/73bd0339/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryBuilder.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryBuilder.java b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryBuilder.java index 1750ac1..18169ca 100644 --- a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryBuilder.java +++ b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryBuilder.java @@ -442,7 +442,9 @@ public class TrieDictionaryBuilder<T> { headOut.writeInt((int) stats.mbpn_footprint); // body size headOut.write(sizeChildOffset); headOut.write(sizeNoValuesBeneath); + positiveShortPreCheck(baseId, "baseId"); headOut.writeShort(baseId); + positiveShortPreCheck(stats.maxValueLength, "stats.maxValueLength"); headOut.writeShort(stats.maxValueLength); headOut.writeUTF(bytesConverter == null ? "" : bytesConverter.getClass().getName()); headOut.close(); @@ -483,6 +485,12 @@ public class TrieDictionaryBuilder<T> { return trieBytes; } + private void positiveShortPreCheck(int i, String fieldName) { + if (!BytesUtil.isPositiveShort(i)) { + throw new IllegalStateException(fieldName + " is not positive short, usually caused by too long dict value."); + } + } + private void build_overwriteChildOffset(int parentOffset, int childOffset, int sizeChildOffset, byte[] trieBytes) { int flags = (int) trieBytes[parentOffset] & (TrieDictionary.BIT_IS_LAST_CHILD | TrieDictionary.BIT_IS_END_OF_VALUE); BytesUtil.writeUnsigned(childOffset, trieBytes, parentOffset, sizeChildOffset);
