minor, stablize org.apache.kylin.dict.TrieDictionaryForestTest#englishWordsTest
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5eedd135 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5eedd135 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5eedd135 Branch: refs/heads/sync Commit: 5eedd135d27da96c2242079fd9e188fce0518f90 Parents: 9198cfe Author: lidongsjtu <lid...@apache.org> Authored: Sun Jan 28 19:55:07 2018 +0800 Committer: lidongsjtu <lid...@apache.org> Committed: Sun Jan 28 19:55:07 2018 +0800 ---------------------------------------------------------------------- .../kylin/dict/TrieDictionaryForestTest.java | 28 ++++++++++++++------ 1 file changed, 20 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/5eedd135/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestTest.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestTest.java b/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestTest.java index 099075c..1f63111 100644 --- a/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestTest.java +++ b/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestTest.java @@ -43,6 +43,7 @@ import java.util.NoSuchElementException; import java.util.Random; import java.util.TreeSet; +import org.apache.kylin.common.util.Bytes; import org.junit.Ignore; import org.junit.Test; @@ -187,8 +188,14 @@ public class TrieDictionaryForestTest { str.add("party"); str.add("parties"); str.add("paint"); - String longStr = "paintjkjdfklajkdljfkdsajklfjklsadjkjekjrklewjrklewjklrjklewjkljkljkljkljweklrjewkljrklewjrlkjewkljrkljkljkjlkjjkljkljkljkljlkjlkjlkjljdfadfads" + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk" + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk" + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk" + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk" + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk" - + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk" + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk"; + String longStr = "paintjkjdfklajkdljfkdsajklfjklsadjkjekjrklewjrklewjklrjklewjkljkljkljkljweklrjewkljrklewjrlkjewkljrkljkljkjlkjjkljkljkljkljlkjlkjlkjljdfadfads" + + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk" + + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk" + + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk" + + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk" + + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk" + + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk" + + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk"; System.out.println("The length of the long string is " + longStr.length()); str.add(longStr); @@ -740,7 +747,8 @@ public class TrieDictionaryForestTest { System.out.println("times:" + i); } - System.out.println("compare build time. Old trie : " + oldDictTotalBuildTime / 1000.0 + "s.New trie : " + newDictTotalBuildTime / 1000.0 + "s"); + System.out.println("compare build time. Old trie : " + oldDictTotalBuildTime / 1000.0 + "s.New trie : " + + newDictTotalBuildTime / 1000.0 + "s"); } private void evaluateDataSize(ArrayList<String> list) { @@ -806,7 +814,8 @@ public class TrieDictionaryForestTest { benchmark("Benchmark", dict, set, map, strArray, array); } - private static int benchmark(String msg, TrieDictionaryForest<String> dict, TreeSet<String> set, HashMap<String, Integer> map, String[] strArray, byte[][] array) { + private static int benchmark(String msg, TrieDictionaryForest<String> dict, TreeSet<String> set, + HashMap<String, Integer> map, String[] strArray, byte[][] array) { int n = set.size(); int times = Math.max(10 * 1000 * 1000 / n, 1); // run 10 million lookups int keep = 0; // make sure JIT don't OPT OUT function calls under test @@ -889,7 +898,7 @@ public class TrieDictionaryForestTest { assertEquals(id, dict.getIdFromValue(value)); assertEquals(value, dict.getValueFromId(id)); - assertArrayEquals(value.getBytes(), dict.getValueByteFromId(id)); + assertArrayEquals(Bytes.toBytes(value), dict.getValueByteFromId(id)); } //test not found value @@ -932,14 +941,16 @@ public class TrieDictionaryForestTest { } public static TrieDictionaryForestBuilder<String> newDictBuilder(Iterable<String> strs, int baseId) { - TrieDictionaryForestBuilder<String> b = new TrieDictionaryForestBuilder<String>(new StringBytesConverter(), baseId); + TrieDictionaryForestBuilder<String> b = new TrieDictionaryForestBuilder<String>(new StringBytesConverter(), + baseId); for (String s : strs) b.addValue(s); return b; } public static TrieDictionaryForestBuilder<String> newDictBuilder(Iterable<String> strs, int baseId, int treeSize) { - TrieDictionaryForestBuilder<String> b = new TrieDictionaryForestBuilder<String>(new StringBytesConverter(), baseId); + TrieDictionaryForestBuilder<String> b = new TrieDictionaryForestBuilder<String>(new StringBytesConverter(), + baseId); b.setMaxTrieTreeSize(treeSize); for (String s : strs) { b.addValue(s); @@ -948,7 +959,8 @@ public class TrieDictionaryForestTest { } public static TrieDictionaryForestBuilder<String> newDictBuilder(Iterator<String> strs, int baseId, int treeSize) { - TrieDictionaryForestBuilder<String> b = new TrieDictionaryForestBuilder<String>(new StringBytesConverter(), baseId); + TrieDictionaryForestBuilder<String> b = new TrieDictionaryForestBuilder<String>(new StringBytesConverter(), + baseId); b.setMaxTrieTreeSize(treeSize); while (strs.hasNext()) b.addValue(strs.next());