Repository: hive Updated Branches: refs/heads/master 3e01ef326 -> 561dbe3b9
HIVE-15682: Eliminate per-row based dummy iterator creation (reviewed by Chao) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/561dbe3b Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/561dbe3b Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/561dbe3b Branch: refs/heads/master Commit: 561dbe3b90bc5cd85a64e22ccd9e384bbf67a782 Parents: 3e01ef3 Author: Xuefu Zhang <xu...@uber.com> Authored: Tue Feb 7 13:48:55 2017 -0800 Committer: Xuefu Zhang <xu...@uber.com> Committed: Tue Feb 7 13:48:55 2017 -0800 ---------------------------------------------------------------------- .../ql/exec/spark/SparkReduceRecordHandler.java | 56 ++++++++++++-------- 1 file changed, 35 insertions(+), 21 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/561dbe3b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkReduceRecordHandler.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkReduceRecordHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkReduceRecordHandler.java index 44f2e4d..8251900 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkReduceRecordHandler.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkReduceRecordHandler.java @@ -211,30 +211,44 @@ public class SparkReduceRecordHandler extends SparkRecordHandler { } /** - * TODO: Instead of creating a dummy iterator per row, we can implement a private method that's - * similar to processRow(Object key, Iterator<E> values) but processes one row at a time. Then, - * we just call that private method here. + * A reusable dummy iterator that has only one value. + * */ - @Override - public void processRow(Object key, final Object value) throws IOException { - processRow(key, new Iterator<Object>() { - boolean done = false; - @Override - public boolean hasNext() { - return !done; - } + private static class DummyIterator implements Iterator<Object> { + private boolean done = false; + private Object value = null; - @Override - public Object next() { - done = true; - return value; - } + public void setValue(Object v) { + this.value = v; + done = false; + } - @Override - public void remove() { - throw new UnsupportedOperationException("Iterator.remove() is not implemented/supported"); - } - }); + @Override + public boolean hasNext() { + return !done; + } + + @Override + public Object next() { + done = true; + return value; + } + + @Override + public void remove() { + throw new UnsupportedOperationException("Iterator.remove() is not implemented/supported"); + } + } + + private DummyIterator dummyIterator = new DummyIterator(); + + /** + * Process one row using a dummy iterator. + */ + @Override + public void processRow(Object key, final Object value) throws IOException { + dummyIterator.setValue(value); + processRow(key, dummyIterator); } @Override
