Repository: kylin Updated Branches: refs/heads/master 0297d9abf -> c04be637d
KYLIN-2794 MultipleDictionaryValueEnumerator should output values in sorted order Signed-off-by: Li Yang <liy...@apache.org> Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b86a7346 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b86a7346 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b86a7346 Branch: refs/heads/master Commit: b86a7346a1fcc3d5cf0215e83eb5279f1c4872a4 Parents: 0297d9a Author: yifei.wu <yifei...@kyligence.io> Authored: Tue Sep 26 15:32:11 2017 +0800 Committer: Li Yang <liy...@apache.org> Committed: Sun Oct 8 18:48:58 2017 +0800 ---------------------------------------------------------------------- .../dict/MultipleDictionaryValueEnumerator.java | 41 +++++++++++++------- .../MultipleDictionaryValueEnumeratorTest.java | 14 +++++++ 2 files changed, 42 insertions(+), 13 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/b86a7346/core-dictionary/src/main/java/org/apache/kylin/dict/MultipleDictionaryValueEnumerator.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/MultipleDictionaryValueEnumerator.java b/core-dictionary/src/main/java/org/apache/kylin/dict/MultipleDictionaryValueEnumerator.java index c1bc5d5..7bd526b 100644 --- a/core-dictionary/src/main/java/org/apache/kylin/dict/MultipleDictionaryValueEnumerator.java +++ b/core-dictionary/src/main/java/org/apache/kylin/dict/MultipleDictionaryValueEnumerator.java @@ -33,13 +33,16 @@ public class MultipleDictionaryValueEnumerator implements IDictionaryValueEnumer private int curDictIndex = 0; private Dictionary<String> curDict; private int curKey; + private List<Integer> curKeys = Lists.newArrayList(); private String curValue = null; private List<Dictionary<String>> dictionaryList; public MultipleDictionaryValueEnumerator(List<DictionaryInfo> dictionaryInfoList) { dictionaryList = Lists.newArrayListWithCapacity(dictionaryInfoList.size()); for (DictionaryInfo dictInfo : dictionaryInfoList) { + Dictionary<String> dictionary = (Dictionary<String>) dictInfo.getDictionaryObject(); dictionaryList.add((Dictionary<String>) dictInfo.getDictionaryObject()); + curKeys.add(dictionary.getMinId()); } if (!dictionaryList.isEmpty()) { curDict = dictionaryList.get(0); @@ -54,22 +57,34 @@ public class MultipleDictionaryValueEnumerator implements IDictionaryValueEnumer @Override public boolean moveNext() throws IOException { - while (curDictIndex < dictionaryList.size()) { - if (curKey <= curDict.getMaxId()) { - curValue = curDict.getValueFromId(curKey); - curKey ++; - - return true; + //initialize the minValue as the first available value in the dictionary list + String minValue = null; + int curDictIndex = 0; + for (int i = 0; i < dictionaryList.size(); i++) { + if (dictionaryList.get(i) == null || curKeys.get(i) >= dictionaryList.get(i).getSize()) { + continue; } - - // move to next dict if exists - if (++curDictIndex < dictionaryList.size()) { - curDict = dictionaryList.get(curDictIndex); - curKey = curDict.getMinId(); + minValue = dictionaryList.get(i).getValueFromId(curKeys.get(i)); + curDictIndex = i; + break; + } + if (minValue == null) { + curValue = null; + return false; + } + // multi-merge dictionary forest + for (int i = 0; i < dictionaryList.size(); i++) { + if (dictionaryList.get(i) == null || curKeys.get(i) >= dictionaryList.get(i).getSize()) { + continue; + } + if (minValue.compareTo(dictionaryList.get(i).getValueFromId(curKeys.get(i))) > 0) { + minValue = dictionaryList.get(i).getValueFromId(curKeys.get(i)); + curDictIndex = i; } } - curValue = null; - return false; + curValue = minValue; + curKeys.set(curDictIndex, curKeys.get(curDictIndex) + 1); + return true; } @Override http://git-wip-us.apache.org/repos/asf/kylin/blob/b86a7346/core-dictionary/src/test/java/org/apache/kylin/dict/MultipleDictionaryValueEnumeratorTest.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/MultipleDictionaryValueEnumeratorTest.java b/core-dictionary/src/test/java/org/apache/kylin/dict/MultipleDictionaryValueEnumeratorTest.java index 2e90bcf..29e9c5d 100644 --- a/core-dictionary/src/test/java/org/apache/kylin/dict/MultipleDictionaryValueEnumeratorTest.java +++ b/core-dictionary/src/test/java/org/apache/kylin/dict/MultipleDictionaryValueEnumeratorTest.java @@ -99,6 +99,20 @@ public class MultipleDictionaryValueEnumeratorTest { assertArrayEquals(new Integer[]{0, 1, 2, 6, 7, 8}, values); } + @Test + public void testUnorderedDicts() throws IOException { + List<DictionaryInfo> dictionaryInfoList = new ArrayList<>(3); + dictionaryInfoList.add(createDictInfo(new int[]{0, 1, 6})); + dictionaryInfoList.add(createDictInfo(new int[]{3, 7, 8})); + dictionaryInfoList.add(createDictInfo(new int[]{2, 7, 9})); + Integer[] values = enumerateDictInfoList(dictionaryInfoList); + assertEquals(9, values.length); + assertArrayEquals(new Integer[]{0, 1, 2, 3, 6, 7, 7, 8, 9}, values); + } + + + + public static class MockDictionary extends Dictionary<String> { private static final long serialVersionUID = 1L;
