Repository: jena Updated Branches: refs/heads/master 44fdb1eb6 -> 6c2c87cd9
Improve performance of DatasetGraphCollection (JENA-813) Switches it to use IteratorConcat instead of IteratorCons. Also deprecates and switches all usages of IteratorCons to IteratorConcat. Project: http://git-wip-us.apache.org/repos/asf/jena/repo Commit: http://git-wip-us.apache.org/repos/asf/jena/commit/e4f6d625 Tree: http://git-wip-us.apache.org/repos/asf/jena/tree/e4f6d625 Diff: http://git-wip-us.apache.org/repos/asf/jena/diff/e4f6d625 Branch: refs/heads/master Commit: e4f6d62508a58e5e2e93be63c53a7c1fd942f25d Parents: 2746da8 Author: Rob Vesse <rve...@apache.org> Authored: Tue Nov 11 10:49:33 2014 +0000 Committer: Rob Vesse <rve...@apache.org> Committed: Tue Nov 11 10:49:33 2014 +0000 ---------------------------------------------------------------------- jena-arq/ReleaseNotes.txt | 5 + .../jena/sparql/core/DatasetGraphBaseFind.java | 2 +- .../jena/sparql/core/DatasetGraphCaching.java | 2 +- .../sparql/core/DatasetGraphCollection.java | 5 +- .../org/apache/jena/atlas/iterator/Iter.java | 29 +++- .../jena/atlas/iterator/IteratorConcat.java | 1 - .../jena/atlas/iterator/IteratorCons.java | 134 ++++++++++--------- .../java/org/apache/jena/atlas/lib/Map2.java | 2 +- .../apache/jena/atlas/iterator/TestIter.java | 24 ++-- .../jena/atlas/iterator/TestIteratorPeek.java | 2 +- 10 files changed, 119 insertions(+), 87 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/jena/blob/e4f6d625/jena-arq/ReleaseNotes.txt ---------------------------------------------------------------------- diff --git a/jena-arq/ReleaseNotes.txt b/jena-arq/ReleaseNotes.txt index 444a93f..ac092c6 100644 --- a/jena-arq/ReleaseNotes.txt +++ b/jena-arq/ReleaseNotes.txt @@ -2,6 +2,11 @@ ChangeLog for ARQ ================= (the list covers new features and refinements. See ASF JIRA for details of bugs fixed) +==== Jena 2.12.2 + ++ JENA-813 : Improve performance of DatasetGraphCollection, deprecate and remove usage of Iter.append() and + IteratorCons in favour of Iter.concat() and IteratorConcat + ==== Jena 2.12.1 + JENA-768 : Add "--out" to the "riot" command to choose a http://git-wip-us.apache.org/repos/asf/jena/blob/e4f6d625/jena-arq/src/main/java/com/hp/hpl/jena/sparql/core/DatasetGraphBaseFind.java ---------------------------------------------------------------------- diff --git a/jena-arq/src/main/java/com/hp/hpl/jena/sparql/core/DatasetGraphBaseFind.java b/jena-arq/src/main/java/com/hp/hpl/jena/sparql/core/DatasetGraphBaseFind.java index e83f0c1..b08fa3a 100644 --- a/jena-arq/src/main/java/com/hp/hpl/jena/sparql/core/DatasetGraphBaseFind.java +++ b/jena-arq/src/main/java/com/hp/hpl/jena/sparql/core/DatasetGraphBaseFind.java @@ -73,7 +73,7 @@ abstract public class DatasetGraphBaseFind extends DatasetGraphBase if ( iter1 == null && iter2 == null ) return Iter.nullIterator() ; // Copes with null in either position. - return Iter.append(iter1, iter2) ; + return Iter.concat(iter1, iter2) ; } protected abstract Iterator<Quad> findInDftGraph(Node s, Node p , Node o) ; http://git-wip-us.apache.org/repos/asf/jena/blob/e4f6d625/jena-arq/src/main/java/com/hp/hpl/jena/sparql/core/DatasetGraphCaching.java ---------------------------------------------------------------------- diff --git a/jena-arq/src/main/java/com/hp/hpl/jena/sparql/core/DatasetGraphCaching.java b/jena-arq/src/main/java/com/hp/hpl/jena/sparql/core/DatasetGraphCaching.java index a73591a..d69383e 100644 --- a/jena-arq/src/main/java/com/hp/hpl/jena/sparql/core/DatasetGraphCaching.java +++ b/jena-arq/src/main/java/com/hp/hpl/jena/sparql/core/DatasetGraphCaching.java @@ -177,7 +177,7 @@ abstract public class DatasetGraphCaching extends DatasetGraphTriplesQuads for ( ; iter.hasNext() ; ) { Node gn = iter.next() ; - quads = Iter.append(quads, findInSpecificNamedGraph(dsg, gn, s, p, o)) ; + quads = Iter.concat(quads, findInSpecificNamedGraph(dsg, gn, s, p, o)) ; } return quads ; } http://git-wip-us.apache.org/repos/asf/jena/blob/e4f6d625/jena-arq/src/main/java/com/hp/hpl/jena/sparql/core/DatasetGraphCollection.java ---------------------------------------------------------------------- diff --git a/jena-arq/src/main/java/com/hp/hpl/jena/sparql/core/DatasetGraphCollection.java b/jena-arq/src/main/java/com/hp/hpl/jena/sparql/core/DatasetGraphCollection.java index 8de21f3..1656469 100644 --- a/jena-arq/src/main/java/com/hp/hpl/jena/sparql/core/DatasetGraphCollection.java +++ b/jena-arq/src/main/java/com/hp/hpl/jena/sparql/core/DatasetGraphCollection.java @@ -21,6 +21,7 @@ package com.hp.hpl.jena.sparql.core; import java.util.Iterator ; import org.apache.jena.atlas.iterator.Iter ; +import org.apache.jena.atlas.iterator.IteratorConcat; import org.apache.jena.atlas.lib.Lib ; import com.hp.hpl.jena.graph.Graph ; @@ -70,7 +71,7 @@ public abstract class DatasetGraphCollection extends DatasetGraphBaseFind protected Iterator<Quad> findInAnyNamedGraphs(Node s, Node p, Node o) { Iterator<Node> gnames = listGraphNodes() ; - Iterator<Quad> iter = null ; + IteratorConcat<Quad> iter = new IteratorConcat<>() ; // Named graphs for ( ; gnames.hasNext() ; ) { @@ -78,7 +79,7 @@ public abstract class DatasetGraphCollection extends DatasetGraphBaseFind Iterator<Quad> qIter = findInSpecificNamedGraph(gn, s, p, o) ; if ( qIter != null ) // copes with null for iter - iter = Iter.append(iter, qIter) ; + iter.add(qIter) ; } return iter ; } http://git-wip-us.apache.org/repos/asf/jena/blob/e4f6d625/jena-arq/src/main/java/org/apache/jena/atlas/iterator/Iter.java ---------------------------------------------------------------------- diff --git a/jena-arq/src/main/java/org/apache/jena/atlas/iterator/Iter.java b/jena-arq/src/main/java/org/apache/jena/atlas/iterator/Iter.java index 52352ea..9ae35eb 100644 --- a/jena-arq/src/main/java/org/apache/jena/atlas/iterator/Iter.java +++ b/jena-arq/src/main/java/org/apache/jena/atlas/iterator/Iter.java @@ -443,11 +443,18 @@ public class Iter<T> implements Iterable<T>, Iterator<T> { return Iter.operate(stream, action) ; } + /** + * @deprecated Use {@link #concat(Iterable, Iterable)} instead which is much more performant + */ + @Deprecated public static <T> Iterator<T> append(Iterable<T> iter1, Iterable<T> iter2) { return IteratorCons.create(iterator(iter1), iterator(iter2)) ; } - // Could try for <? extends T> on each arg. + /** + * @deprecated Use {@link #concat(Iterator, Iterator)} instead which is much more performant + */ + @Deprecated public static <T> Iterator<T> append(Iterator<? extends T> iter1, Iterator<? extends T> iter2) { return IteratorCons.create(iter1, iter2) ; } @@ -747,7 +754,7 @@ public class Iter<T> implements Iterable<T>, Iterator<T> { return iter2 ; if ( iter2 == null ) return iter1 ; - return iter1.append(iter2) ; + return iter1.concat(iter2) ; } public static <T> Iterator<T> concat(Iterator<T> iter1, Iterator<T> iter2) { @@ -755,7 +762,15 @@ public class Iter<T> implements Iterable<T>, Iterator<T> { return iter2 ; if ( iter2 == null ) return iter1 ; - return Iter.iter(iter1).append(Iter.iter(iter2)) ; + return Iter.iter(iter1).concat(Iter.iter(iter2)) ; + } + + public static <T> Iterator<T> concat(Iterable<T> iter1, Iterable<T> iter2) { + if (iter1 == null) + return iter2.iterator(); + if (iter2 == null) + return iter1.iterator(); + return Iter.concat(iter1.iterator(), iter2.iterator()); } public static <T> T first(Iterator<T> iter, Filter<T> filter) { @@ -886,9 +901,17 @@ public class Iter<T> implements Iterable<T>, Iterator<T> { apply(iterator, action) ; } + /** + * @deprecated Use {@link #concat(Iterator)} instead which is much more performant + */ + @Deprecated public Iter<T> append(Iterator<T> iter) { return new Iter<>(IteratorCons.create(iterator, iter)) ; } + + public Iter<T> concat(Iterator<T> iter) { + return new Iter<>(IteratorConcat.concat(iterator, iter)); + } /** Return an Iter that yields at most the first N items */ public Iter<T> take(int N) { http://git-wip-us.apache.org/repos/asf/jena/blob/e4f6d625/jena-arq/src/main/java/org/apache/jena/atlas/iterator/IteratorConcat.java ---------------------------------------------------------------------- diff --git a/jena-arq/src/main/java/org/apache/jena/atlas/iterator/IteratorConcat.java b/jena-arq/src/main/java/org/apache/jena/atlas/iterator/IteratorConcat.java index 34badf9..3860d0c 100644 --- a/jena-arq/src/main/java/org/apache/jena/atlas/iterator/IteratorConcat.java +++ b/jena-arq/src/main/java/org/apache/jena/atlas/iterator/IteratorConcat.java @@ -37,7 +37,6 @@ public class IteratorConcat<T> implements Iterator<T> private Iterator<T> removeFrom = null ; boolean finished = false ; - /** @see IteratorCons */ public static <T> Iterator<T> concat(Iterator<T> iter1, Iterator<T> iter2) { if (iter2 == null) return iter1 ; http://git-wip-us.apache.org/repos/asf/jena/blob/e4f6d625/jena-arq/src/main/java/org/apache/jena/atlas/iterator/IteratorCons.java ---------------------------------------------------------------------- diff --git a/jena-arq/src/main/java/org/apache/jena/atlas/iterator/IteratorCons.java b/jena-arq/src/main/java/org/apache/jena/atlas/iterator/IteratorCons.java index b255f59..e86cdee 100644 --- a/jena-arq/src/main/java/org/apache/jena/atlas/iterator/IteratorCons.java +++ b/jena-arq/src/main/java/org/apache/jena/atlas/iterator/IteratorCons.java @@ -18,100 +18,104 @@ package org.apache.jena.atlas.iterator; -import java.util.Iterator ; -import java.util.NoSuchElementException ; +import java.util.Iterator; +import java.util.NoSuchElementException; -/** IteratorCons : the concatenation of two iterators. +/** + * IteratorCons : the concatenation of two iterators. + * <p> + * This has known performance issues when used with lots of iterators and so + * should be avoided in favour of {@link IteratorConcat} which is much better + * performing (see <a + * href="https://issues.apache.org/jira/browse/JENA-813">JENA-813</a> for some + * discussion) + * </p> + * + * @deprecated Use the more performant {@link IteratorConcat} instead */ - -public class IteratorCons<T> implements Iterator<T>, Iterable<T> -{ +@Deprecated +public class IteratorCons<T> implements Iterator<T>, Iterable<T> { // No - we don't really need IteratorCons and IteratorConcat // Historical. - - private Iterator<? extends T> iter1 ; - private Iterator<? extends T> iter2 ; - private Iterator<? extends T> removeFrom ; - public static <X> Iterator<X> create(Iterator<? extends X> iter1, Iterator<? extends X> iter2) - { - if ( iter1 == null && iter2 == null ) - return Iter.nullIter() ; - - // The casts are safe because an iterator can only return X, and does not take an X an an assignment. - if ( iter1 == null ) - { + private Iterator<? extends T> iter1; + private Iterator<? extends T> iter2; + private Iterator<? extends T> removeFrom; + + /** + * @deprecated Use {@link IteratorConcat#concat(Iterator, Iterator)} instead which is much more performant + */ + @Deprecated + public static <X> Iterator<X> create(Iterator<? extends X> iter1, Iterator<? extends X> iter2) { + if (iter1 == null && iter2 == null) + return Iter.nullIter(); + + // The casts are safe because an iterator can only return X, and does + // not take an X an an assignment. + if (iter1 == null) { @SuppressWarnings("unchecked") - Iterator<X> x = (Iterator<X>)iter2 ; - return x ; + Iterator<X> x = (Iterator<X>) iter2; + return x; } - - if ( iter2 == null ) - { + + if (iter2 == null) { @SuppressWarnings("unchecked") - Iterator<X> x = (Iterator<X>)iter1 ; - return x ; + Iterator<X> x = (Iterator<X>) iter1; + return x; } - - return new IteratorCons<>(iter1, iter2) ; + + return new IteratorCons<>(iter1, iter2); } - - private IteratorCons(Iterator<? extends T> iter1, Iterator<? extends T> iter2) - { - this.iter1 = iter1 ; - this.iter2 = iter2 ; + + private IteratorCons(Iterator<? extends T> iter1, Iterator<? extends T> iter2) { + this.iter1 = iter1; + this.iter2 = iter2; } @Override - public boolean hasNext() - { - if ( iter1 != null ) - { - if ( iter1.hasNext() ) return true ; + public boolean hasNext() { + if (iter1 != null) { + if (iter1.hasNext()) + return true; // Iter1 ends - iter1 = null ; + iter1 = null; } - - if ( iter2 != null ) - { - if ( iter2.hasNext() ) return true ; + + if (iter2 != null) { + if (iter2.hasNext()) + return true; // Iter2 ends - iter2 = null ; + iter2 = null; } - return false ; + return false; } @Override - public T next() - { - if ( ! hasNext() ) - throw new NoSuchElementException("Iterator2.next") ; - if ( iter1 != null ) - { - removeFrom = iter1 ; + public T next() { + if (!hasNext()) + throw new NoSuchElementException("Iterator2.next"); + if (iter1 != null) { + removeFrom = iter1; return iter1.next(); } - if ( iter2 != null ) - { - removeFrom = iter2 ; + if (iter2 != null) { + removeFrom = iter2; return iter2.next(); } - throw new Error("Iterator2.next") ; + throw new Error("Iterator2.next"); } @Override - public void remove() - { - if ( null == removeFrom ) - throw new IllegalStateException("no calls to next() since last call to remove()") ; - - removeFrom.remove() ; - removeFrom = null ; + public void remove() { + if (null == removeFrom) + throw new IllegalStateException("no calls to next() since last call to remove()"); + + removeFrom.remove(); + removeFrom = null; } @Override - public Iterator<T> iterator() - { - return this ; + public Iterator<T> iterator() { + return this; } } http://git-wip-us.apache.org/repos/asf/jena/blob/e4f6d625/jena-arq/src/main/java/org/apache/jena/atlas/lib/Map2.java ---------------------------------------------------------------------- diff --git a/jena-arq/src/main/java/org/apache/jena/atlas/lib/Map2.java b/jena-arq/src/main/java/org/apache/jena/atlas/lib/Map2.java index 75697e0..3e3a254 100644 --- a/jena-arq/src/main/java/org/apache/jena/atlas/lib/Map2.java +++ b/jena-arq/src/main/java/org/apache/jena/atlas/lib/Map2.java @@ -81,7 +81,7 @@ public class Map2<K, V> implements Iterable<K> Iter<K> iter1 = Iter.iter(map1.keySet().iterator()) ; if ( map2 == null ) return iter1 ; - return iter1.append(map2.iterator()) ; + return iter1.concat(Iter.iter(map2.iterator())) ; } public boolean isEmpty() http://git-wip-us.apache.org/repos/asf/jena/blob/e4f6d625/jena-arq/src/test/java/org/apache/jena/atlas/iterator/TestIter.java ---------------------------------------------------------------------- diff --git a/jena-arq/src/test/java/org/apache/jena/atlas/iterator/TestIter.java b/jena-arq/src/test/java/org/apache/jena/atlas/iterator/TestIter.java index f611c03..a8b9937 100644 --- a/jena-arq/src/test/java/org/apache/jena/atlas/iterator/TestIter.java +++ b/jena-arq/src/test/java/org/apache/jena/atlas/iterator/TestIter.java @@ -37,29 +37,29 @@ public class TestIter List<String> data3 = Arrays.asList(null, "x", null, null, null, "y", "z", null); @Test - public void append_1() + public void concat_1() { - Iterator<String> iter = Iter.append(data1, data0) ; + Iterator<String> iter = Iter.concat(data1, data0) ; test(iter, "a") ; } @Test - public void append_2() + public void concat_2() { - Iterator<String> iter = Iter.append(data0, data1) ; + Iterator<String> iter = Iter.concat(data0, data1) ; test(iter, "a") ; } @Test - public void append_3() + public void concat_3() { - Iterator<String> iter = Iter.append(data1, data2) ; + Iterator<String> iter = Iter.concat(data1, data2) ; test(iter, "a", "x", "y", "z") ; } @Test - public void append_4() + public void concat_4() { List<String> L = new ArrayList<>(3); L.add("a"); @@ -71,7 +71,7 @@ public class TestIter R.add("f"); - Iterator<String> LR = Iter.append(L, R) ; + Iterator<String> LR = Iter.concat(L, R) ; while (LR.hasNext()) { @@ -89,7 +89,7 @@ public class TestIter } @Test - public void append_5() + public void concat_5() { List<String> L = new ArrayList<>(3); L.add("a"); @@ -101,7 +101,7 @@ public class TestIter R.add("f"); - Iterator<String> LR = Iter.append(L, R) ; + Iterator<String> LR = Iter.concat(L, R) ; while (LR.hasNext()) { @@ -119,7 +119,7 @@ public class TestIter } @Test - public void append_6() + public void concat_6() { List<String> L = new ArrayList<>(3); L.add("a"); @@ -131,7 +131,7 @@ public class TestIter R.add("f"); - Iterator<String> LR = Iter.append(L, R) ; + Iterator<String> LR = Iter.concat(L, R) ; while (LR.hasNext()) { http://git-wip-us.apache.org/repos/asf/jena/blob/e4f6d625/jena-arq/src/test/java/org/apache/jena/atlas/iterator/TestIteratorPeek.java ---------------------------------------------------------------------- diff --git a/jena-arq/src/test/java/org/apache/jena/atlas/iterator/TestIteratorPeek.java b/jena-arq/src/test/java/org/apache/jena/atlas/iterator/TestIteratorPeek.java index 33f0b70..59b3dee 100644 --- a/jena-arq/src/test/java/org/apache/jena/atlas/iterator/TestIteratorPeek.java +++ b/jena-arq/src/test/java/org/apache/jena/atlas/iterator/TestIteratorPeek.java @@ -45,7 +45,7 @@ public class TestIteratorPeek extends BaseTest @Test public void iter_01() { Iter<String> iter = Iter.iter(data2) ; - iter = iter.append(data2.iterator()) ; + iter = iter.concat(data2.iterator()) ; test(iter, "x", "y", "z", "x", "y", "z") ; }