[36/50] [abbrv] kylin git commit: KYLIN-2248 TopN merge further optimization after KYLIN-1917

2016-12-06 Thread lidong 
KYLIN-2248 TopN merge further optimization after KYLIN-1917


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/59a30f66
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/59a30f66
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/59a30f66

Branch: refs/heads/master-cdh5.7
Commit: 59a30f66d47cc1838e6852405699fd7957bfac29
Parents: af429e5
Author: shaofengshi 
Authored: Sun Dec 4 09:39:45 2016 +0800
Committer: shaofengshi 
Committed: Mon Dec 5 17:42:34 2016 +0800

--
 .../apache/kylin/measure/topn/TopNCounter.java  | 47 ++--
 1 file changed, 13 insertions(+), 34 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/59a30f66/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNCounter.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNCounter.java 
b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNCounter.java
index 968e694..caf7961 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNCounter.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNCounter.java
@@ -26,11 +26,9 @@ import java.util.Iterator;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
-import java.util.Set;
 
-import com.google.common.collect.Maps;
 import com.google.common.collect.Lists;
-import com.google.common.collect.Sets;
+import com.google.common.collect.Maps;
 
 /**
  * Modified from the StreamSummary.java in 
https://github.com/addthis/stream-lib
@@ -157,41 +155,22 @@ public class TopNCounter implements 
Iterable {
  * @return
  */
 public TopNCounter merge(TopNCounter another) {
-double m1 = 0.0, m2 = 0.0;
-if (this.size() >= this.capacity) {
-m1 = this.counterList.getLast().count;
-}
-
-if (another.size() >= another.capacity) {
-m2 = another.counterList.getLast().count;
-}
-
-Set duplicateItems = Sets.newHashSet();
-List notDuplicateItems = Lists.newArrayList();
-
-for (Map.Entry entry : this.counterMap.entrySet()) {
-T item = entry.getKey();
-Counter existing = another.counterMap.get(item);
-if (existing != null) {
-duplicateItems.add(item);
-} else {
-notDuplicateItems.add(item);
+boolean thisFull = this.size() >= this.capacity;
+boolean anotherFull = another.size() >= another.capacity;
+double m1 = thisFull ? this.counterList.getLast().count : 0.0;
+double m2 = anotherFull ? another.counterList.getLast().count : 0.0;
+
+if (anotherFull == true) {
+for (Counter entry : this.counterMap.values()) {
+entry.count += m2;
 }
 }
 
-for (T item : duplicateItems) {
-this.offer(item, another.counterMap.get(item).count);
-}
-
-for (T item : notDuplicateItems) {
-this.offer(item, m2);
-}
-
 for (Map.Entry entry : another.counterMap.entrySet()) {
-T item = entry.getKey();
-if (duplicateItems.contains(item) == false) {
-double counter = entry.getValue().count;
-this.offer(item, counter + m1);
+if (this.counterMap.containsKey(entry.getKey())) {
+this.offer(entry.getValue().getItem(), (entry.getValue().count 
- m2));
+} else {
+this.offer(entry.getValue().getItem(), entry.getValue().count 
+ m1);
 }
 }

[37/50] [abbrv] kylin git commit: KYLIN-2248 TopN merge further optimization after KYLIN-1917

2016-12-06 Thread lidong 
KYLIN-2248 TopN merge further optimization after KYLIN-1917


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/59a30f66
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/59a30f66
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/59a30f66

Branch: refs/heads/master-hbase1.x
Commit: 59a30f66d47cc1838e6852405699fd7957bfac29
Parents: af429e5
Author: shaofengshi 
Authored: Sun Dec 4 09:39:45 2016 +0800
Committer: shaofengshi 
Committed: Mon Dec 5 17:42:34 2016 +0800

--
 .../apache/kylin/measure/topn/TopNCounter.java  | 47 ++--
 1 file changed, 13 insertions(+), 34 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/59a30f66/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNCounter.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNCounter.java 
b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNCounter.java
index 968e694..caf7961 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNCounter.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNCounter.java
@@ -26,11 +26,9 @@ import java.util.Iterator;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
-import java.util.Set;
 
-import com.google.common.collect.Maps;
 import com.google.common.collect.Lists;
-import com.google.common.collect.Sets;
+import com.google.common.collect.Maps;
 
 /**
  * Modified from the StreamSummary.java in 
https://github.com/addthis/stream-lib
@@ -157,41 +155,22 @@ public class TopNCounter implements 
Iterable {
  * @return
  */
 public TopNCounter merge(TopNCounter another) {
-double m1 = 0.0, m2 = 0.0;
-if (this.size() >= this.capacity) {
-m1 = this.counterList.getLast().count;
-}
-
-if (another.size() >= another.capacity) {
-m2 = another.counterList.getLast().count;
-}
-
-Set duplicateItems = Sets.newHashSet();
-List notDuplicateItems = Lists.newArrayList();
-
-for (Map.Entry entry : this.counterMap.entrySet()) {
-T item = entry.getKey();
-Counter existing = another.counterMap.get(item);
-if (existing != null) {
-duplicateItems.add(item);
-} else {
-notDuplicateItems.add(item);
+boolean thisFull = this.size() >= this.capacity;
+boolean anotherFull = another.size() >= another.capacity;
+double m1 = thisFull ? this.counterList.getLast().count : 0.0;
+double m2 = anotherFull ? another.counterList.getLast().count : 0.0;
+
+if (anotherFull == true) {
+for (Counter entry : this.counterMap.values()) {
+entry.count += m2;
 }
 }
 
-for (T item : duplicateItems) {
-this.offer(item, another.counterMap.get(item).count);
-}
-
-for (T item : notDuplicateItems) {
-this.offer(item, m2);
-}
-
 for (Map.Entry entry : another.counterMap.entrySet()) {
-T item = entry.getKey();
-if (duplicateItems.contains(item) == false) {
-double counter = entry.getValue().count;
-this.offer(item, counter + m1);
+if (this.counterMap.containsKey(entry.getKey())) {
+this.offer(entry.getValue().getItem(), (entry.getValue().count 
- m2));
+} else {
+this.offer(entry.getValue().getItem(), entry.getValue().count 
+ m1);
 }
 }

kylin git commit: KYLIN-2248 TopN merge further optimization after KYLIN-1917

2016-12-05 Thread shaofengshi 
Repository: kylin
Updated Branches:
  refs/heads/master af429e5cb -> 59a30f66d


KYLIN-2248 TopN merge further optimization after KYLIN-1917


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/59a30f66
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/59a30f66
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/59a30f66

Branch: refs/heads/master
Commit: 59a30f66d47cc1838e6852405699fd7957bfac29
Parents: af429e5
Author: shaofengshi 
Authored: Sun Dec 4 09:39:45 2016 +0800
Committer: shaofengshi 
Committed: Mon Dec 5 17:42:34 2016 +0800

--
 .../apache/kylin/measure/topn/TopNCounter.java  | 47 ++--
 1 file changed, 13 insertions(+), 34 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/59a30f66/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNCounter.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNCounter.java 
b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNCounter.java
index 968e694..caf7961 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNCounter.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNCounter.java
@@ -26,11 +26,9 @@ import java.util.Iterator;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
-import java.util.Set;
 
-import com.google.common.collect.Maps;
 import com.google.common.collect.Lists;
-import com.google.common.collect.Sets;
+import com.google.common.collect.Maps;
 
 /**
  * Modified from the StreamSummary.java in 
https://github.com/addthis/stream-lib
@@ -157,41 +155,22 @@ public class TopNCounter implements 
Iterable {
  * @return
  */
 public TopNCounter merge(TopNCounter another) {
-double m1 = 0.0, m2 = 0.0;
-if (this.size() >= this.capacity) {
-m1 = this.counterList.getLast().count;
-}
-
-if (another.size() >= another.capacity) {
-m2 = another.counterList.getLast().count;
-}
-
-Set duplicateItems = Sets.newHashSet();
-List notDuplicateItems = Lists.newArrayList();
-
-for (Map.Entry entry : this.counterMap.entrySet()) {
-T item = entry.getKey();
-Counter existing = another.counterMap.get(item);
-if (existing != null) {
-duplicateItems.add(item);
-} else {
-notDuplicateItems.add(item);
+boolean thisFull = this.size() >= this.capacity;
+boolean anotherFull = another.size() >= another.capacity;
+double m1 = thisFull ? this.counterList.getLast().count : 0.0;
+double m2 = anotherFull ? another.counterList.getLast().count : 0.0;
+
+if (anotherFull == true) {
+for (Counter entry : this.counterMap.values()) {
+entry.count += m2;
 }
 }
 
-for (T item : duplicateItems) {
-this.offer(item, another.counterMap.get(item).count);
-}
-
-for (T item : notDuplicateItems) {
-this.offer(item, m2);
-}
-
 for (Map.Entry entry : another.counterMap.entrySet()) {
-T item = entry.getKey();
-if (duplicateItems.contains(item) == false) {
-double counter = entry.getValue().count;
-this.offer(item, counter + m1);
+if (this.counterMap.containsKey(entry.getKey())) {
+this.offer(entry.getValue().getItem(), (entry.getValue().count 
- m2));
+} else {
+this.offer(entry.getValue().getItem(), entry.getValue().count 
+ m1);
 }
 }