jena-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From rve...@apache.org
Subject [1/2] jena git commit: Improve performance of DatasetGraphCollection (JENA-813)
Date Tue, 11 Nov 2014 10:55:34 GMT
Repository: jena
Updated Branches:
  refs/heads/master 44fdb1eb6 -> 6c2c87cd9


Improve performance of DatasetGraphCollection (JENA-813)

Switches it to use IteratorConcat instead of IteratorCons.  Also
deprecates and switches all usages of IteratorCons to IteratorConcat.


Project: http://git-wip-us.apache.org/repos/asf/jena/repo
Commit: http://git-wip-us.apache.org/repos/asf/jena/commit/e4f6d625
Tree: http://git-wip-us.apache.org/repos/asf/jena/tree/e4f6d625
Diff: http://git-wip-us.apache.org/repos/asf/jena/diff/e4f6d625

Branch: refs/heads/master
Commit: e4f6d62508a58e5e2e93be63c53a7c1fd942f25d
Parents: 2746da8
Author: Rob Vesse <rvesse@apache.org>
Authored: Tue Nov 11 10:49:33 2014 +0000
Committer: Rob Vesse <rvesse@apache.org>
Committed: Tue Nov 11 10:49:33 2014 +0000

----------------------------------------------------------------------
 jena-arq/ReleaseNotes.txt                       |   5 +
 .../jena/sparql/core/DatasetGraphBaseFind.java  |   2 +-
 .../jena/sparql/core/DatasetGraphCaching.java   |   2 +-
 .../sparql/core/DatasetGraphCollection.java     |   5 +-
 .../org/apache/jena/atlas/iterator/Iter.java    |  29 +++-
 .../jena/atlas/iterator/IteratorConcat.java     |   1 -
 .../jena/atlas/iterator/IteratorCons.java       | 134 ++++++++++---------
 .../java/org/apache/jena/atlas/lib/Map2.java    |   2 +-
 .../apache/jena/atlas/iterator/TestIter.java    |  24 ++--
 .../jena/atlas/iterator/TestIteratorPeek.java   |   2 +-
 10 files changed, 119 insertions(+), 87 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/jena/blob/e4f6d625/jena-arq/ReleaseNotes.txt
----------------------------------------------------------------------
diff --git a/jena-arq/ReleaseNotes.txt b/jena-arq/ReleaseNotes.txt
index 444a93f..ac092c6 100644
--- a/jena-arq/ReleaseNotes.txt
+++ b/jena-arq/ReleaseNotes.txt
@@ -2,6 +2,11 @@ ChangeLog for ARQ
 =================
 (the list covers new features and refinements.  See ASF JIRA for details of bugs fixed) 
 
+==== Jena 2.12.2
+
++ JENA-813 : Improve performance of DatasetGraphCollection, deprecate and remove usage of
Iter.append() and
+             IteratorCons in favour of Iter.concat() and IteratorConcat
+
 ==== Jena 2.12.1
 
 + JENA-768 : Add "--out" to the "riot" command to choose a 

http://git-wip-us.apache.org/repos/asf/jena/blob/e4f6d625/jena-arq/src/main/java/com/hp/hpl/jena/sparql/core/DatasetGraphBaseFind.java
----------------------------------------------------------------------
diff --git a/jena-arq/src/main/java/com/hp/hpl/jena/sparql/core/DatasetGraphBaseFind.java
b/jena-arq/src/main/java/com/hp/hpl/jena/sparql/core/DatasetGraphBaseFind.java
index e83f0c1..b08fa3a 100644
--- a/jena-arq/src/main/java/com/hp/hpl/jena/sparql/core/DatasetGraphBaseFind.java
+++ b/jena-arq/src/main/java/com/hp/hpl/jena/sparql/core/DatasetGraphBaseFind.java
@@ -73,7 +73,7 @@ abstract public class DatasetGraphBaseFind extends DatasetGraphBase
         if ( iter1 == null && iter2 == null )
             return Iter.nullIterator() ;
         // Copes with null in either position.
-        return Iter.append(iter1, iter2) ;
+        return Iter.concat(iter1, iter2) ;
     }
 
     protected abstract Iterator<Quad> findInDftGraph(Node s, Node p , Node o) ;

http://git-wip-us.apache.org/repos/asf/jena/blob/e4f6d625/jena-arq/src/main/java/com/hp/hpl/jena/sparql/core/DatasetGraphCaching.java
----------------------------------------------------------------------
diff --git a/jena-arq/src/main/java/com/hp/hpl/jena/sparql/core/DatasetGraphCaching.java b/jena-arq/src/main/java/com/hp/hpl/jena/sparql/core/DatasetGraphCaching.java
index a73591a..d69383e 100644
--- a/jena-arq/src/main/java/com/hp/hpl/jena/sparql/core/DatasetGraphCaching.java
+++ b/jena-arq/src/main/java/com/hp/hpl/jena/sparql/core/DatasetGraphCaching.java
@@ -177,7 +177,7 @@ abstract public class DatasetGraphCaching extends DatasetGraphTriplesQuads
             for ( ; iter.hasNext() ; )
             {
                 Node gn = iter.next() ;
-                quads = Iter.append(quads, findInSpecificNamedGraph(dsg, gn, s, p, o)) ;
+                quads = Iter.concat(quads, findInSpecificNamedGraph(dsg, gn, s, p, o)) ;
             }
             return quads ;
         }

http://git-wip-us.apache.org/repos/asf/jena/blob/e4f6d625/jena-arq/src/main/java/com/hp/hpl/jena/sparql/core/DatasetGraphCollection.java
----------------------------------------------------------------------
diff --git a/jena-arq/src/main/java/com/hp/hpl/jena/sparql/core/DatasetGraphCollection.java
b/jena-arq/src/main/java/com/hp/hpl/jena/sparql/core/DatasetGraphCollection.java
index 8de21f3..1656469 100644
--- a/jena-arq/src/main/java/com/hp/hpl/jena/sparql/core/DatasetGraphCollection.java
+++ b/jena-arq/src/main/java/com/hp/hpl/jena/sparql/core/DatasetGraphCollection.java
@@ -21,6 +21,7 @@ package com.hp.hpl.jena.sparql.core;
 import java.util.Iterator ;
 
 import org.apache.jena.atlas.iterator.Iter ;
+import org.apache.jena.atlas.iterator.IteratorConcat;
 import org.apache.jena.atlas.lib.Lib ;
 
 import com.hp.hpl.jena.graph.Graph ;
@@ -70,7 +71,7 @@ public abstract class DatasetGraphCollection extends DatasetGraphBaseFind
     protected Iterator<Quad> findInAnyNamedGraphs(Node s, Node p, Node o)
     {
         Iterator<Node> gnames = listGraphNodes() ;
-        Iterator<Quad> iter = null ;
+        IteratorConcat<Quad> iter = new IteratorConcat<>() ;
         // Named graphs
         for ( ; gnames.hasNext() ; )  
         {
@@ -78,7 +79,7 @@ public abstract class DatasetGraphCollection extends DatasetGraphBaseFind
             Iterator<Quad> qIter = findInSpecificNamedGraph(gn, s, p, o) ;
             if ( qIter != null )
                 // copes with null for iter
-                iter = Iter.append(iter, qIter) ;
+                iter.add(qIter) ;
         }
         return iter ;
     }

http://git-wip-us.apache.org/repos/asf/jena/blob/e4f6d625/jena-arq/src/main/java/org/apache/jena/atlas/iterator/Iter.java
----------------------------------------------------------------------
diff --git a/jena-arq/src/main/java/org/apache/jena/atlas/iterator/Iter.java b/jena-arq/src/main/java/org/apache/jena/atlas/iterator/Iter.java
index 52352ea..9ae35eb 100644
--- a/jena-arq/src/main/java/org/apache/jena/atlas/iterator/Iter.java
+++ b/jena-arq/src/main/java/org/apache/jena/atlas/iterator/Iter.java
@@ -443,11 +443,18 @@ public class Iter<T> implements Iterable<T>, Iterator<T>
{
         return Iter.operate(stream, action) ;
     }
 
+    /**
+     * @deprecated Use {@link #concat(Iterable, Iterable)} instead which is much more performant
+     */
+    @Deprecated
     public static <T> Iterator<T> append(Iterable<T> iter1, Iterable<T>
iter2) {
         return IteratorCons.create(iterator(iter1), iterator(iter2)) ;
     }
 
-    // Could try for <? extends T> on each arg.
+    /**
+     * @deprecated Use {@link #concat(Iterator, Iterator)} instead which is much more performant
+     */
+    @Deprecated
     public static <T> Iterator<T> append(Iterator<? extends T> iter1, Iterator<?
extends T> iter2) {
         return IteratorCons.create(iter1, iter2) ;
     }
@@ -747,7 +754,7 @@ public class Iter<T> implements Iterable<T>, Iterator<T>
{
             return iter2 ;
         if ( iter2 == null )
             return iter1 ;
-        return iter1.append(iter2) ;
+        return iter1.concat(iter2) ;
     }
 
     public static <T> Iterator<T> concat(Iterator<T> iter1, Iterator<T>
iter2) {
@@ -755,7 +762,15 @@ public class Iter<T> implements Iterable<T>, Iterator<T>
{
             return iter2 ;
         if ( iter2 == null )
             return iter1 ;
-        return Iter.iter(iter1).append(Iter.iter(iter2)) ;
+        return Iter.iter(iter1).concat(Iter.iter(iter2)) ;
+    }
+    
+    public static <T> Iterator<T> concat(Iterable<T> iter1, Iterable<T>
iter2) {
+        if (iter1 == null)
+            return iter2.iterator();
+        if (iter2 == null)
+            return iter1.iterator();
+        return Iter.concat(iter1.iterator(), iter2.iterator());
     }
 
     public static <T> T first(Iterator<T> iter, Filter<T> filter) {
@@ -886,9 +901,17 @@ public class Iter<T> implements Iterable<T>, Iterator<T>
{
         apply(iterator, action) ;
     }
 
+    /**
+     * @deprecated Use {@link #concat(Iterator)} instead which is much more performant
+     */
+    @Deprecated
     public Iter<T> append(Iterator<T> iter) {
         return new Iter<>(IteratorCons.create(iterator, iter)) ;
     }
+    
+    public Iter<T> concat(Iterator<T> iter) {
+        return new Iter<>(IteratorConcat.concat(iterator, iter));
+    }
 
     /** Return an Iter that yields at most the first N items */
     public Iter<T> take(int N) {

http://git-wip-us.apache.org/repos/asf/jena/blob/e4f6d625/jena-arq/src/main/java/org/apache/jena/atlas/iterator/IteratorConcat.java
----------------------------------------------------------------------
diff --git a/jena-arq/src/main/java/org/apache/jena/atlas/iterator/IteratorConcat.java b/jena-arq/src/main/java/org/apache/jena/atlas/iterator/IteratorConcat.java
index 34badf9..3860d0c 100644
--- a/jena-arq/src/main/java/org/apache/jena/atlas/iterator/IteratorConcat.java
+++ b/jena-arq/src/main/java/org/apache/jena/atlas/iterator/IteratorConcat.java
@@ -37,7 +37,6 @@ public class IteratorConcat<T> implements Iterator<T>
     private Iterator<T> removeFrom = null ;
     boolean finished = false ;
     
-    /** @see IteratorCons */
     public static <T> Iterator<T> concat(Iterator<T> iter1, Iterator<T>
iter2)
     {
         if (iter2 == null) return iter1 ;

http://git-wip-us.apache.org/repos/asf/jena/blob/e4f6d625/jena-arq/src/main/java/org/apache/jena/atlas/iterator/IteratorCons.java
----------------------------------------------------------------------
diff --git a/jena-arq/src/main/java/org/apache/jena/atlas/iterator/IteratorCons.java b/jena-arq/src/main/java/org/apache/jena/atlas/iterator/IteratorCons.java
index b255f59..e86cdee 100644
--- a/jena-arq/src/main/java/org/apache/jena/atlas/iterator/IteratorCons.java
+++ b/jena-arq/src/main/java/org/apache/jena/atlas/iterator/IteratorCons.java
@@ -18,100 +18,104 @@
 
 package org.apache.jena.atlas.iterator;
 
-import java.util.Iterator ;
-import java.util.NoSuchElementException ;
+import java.util.Iterator;
+import java.util.NoSuchElementException;
 
-/** IteratorCons : the concatenation of two iterators.
+/**
+ * IteratorCons : the concatenation of two iterators.
+ * <p>
+ * This has known performance issues when used with lots of iterators and so
+ * should be avoided in favour of {@link IteratorConcat} which is much better
+ * performing (see <a
+ * href="https://issues.apache.org/jira/browse/JENA-813">JENA-813</a> for some
+ * discussion)
+ * </p>
+ * 
+ * @deprecated Use the more performant {@link IteratorConcat} instead
  */
-
-public class IteratorCons<T> implements Iterator<T>, Iterable<T>
-{
+@Deprecated
+public class IteratorCons<T> implements Iterator<T>, Iterable<T> {
     // No - we don't really need IteratorCons and IteratorConcat
     // Historical.
-    
-    private Iterator<? extends T> iter1 ;
-    private Iterator<? extends T> iter2 ;
-    private Iterator<? extends T> removeFrom ;
 
-    public static <X> Iterator<X> create(Iterator<? extends X> iter1, Iterator<?
extends X> iter2)
-    {
-        if ( iter1 == null && iter2 == null )
-            return Iter.nullIter() ;
-        
-        // The casts are safe because an iterator can only return X, and does not take an
X an an assignment.  
-        if ( iter1 == null )
-        {
+    private Iterator<? extends T> iter1;
+    private Iterator<? extends T> iter2;
+    private Iterator<? extends T> removeFrom;
+
+    /**
+     * @deprecated Use {@link IteratorConcat#concat(Iterator, Iterator)} instead which is
much more performant
+     */
+    @Deprecated
+    public static <X> Iterator<X> create(Iterator<? extends X> iter1, Iterator<?
extends X> iter2) {
+        if (iter1 == null && iter2 == null)
+            return Iter.nullIter();
+
+        // The casts are safe because an iterator can only return X, and does
+        // not take an X an an assignment.
+        if (iter1 == null) {
             @SuppressWarnings("unchecked")
-            Iterator<X> x = (Iterator<X>)iter2 ;
-            return x ;
+            Iterator<X> x = (Iterator<X>) iter2;
+            return x;
         }
-        
-        if ( iter2 == null )
-        {
+
+        if (iter2 == null) {
             @SuppressWarnings("unchecked")
-            Iterator<X> x = (Iterator<X>)iter1 ;
-            return x ;
+            Iterator<X> x = (Iterator<X>) iter1;
+            return x;
         }
-        
-        return new IteratorCons<>(iter1, iter2) ;
+
+        return new IteratorCons<>(iter1, iter2);
     }
-    
-    private IteratorCons(Iterator<? extends T> iter1, Iterator<? extends T> iter2)
-    {
-        this.iter1 = iter1 ;
-        this.iter2 = iter2 ;
+
+    private IteratorCons(Iterator<? extends T> iter1, Iterator<? extends T> iter2)
{
+        this.iter1 = iter1;
+        this.iter2 = iter2;
     }
 
     @Override
-    public boolean hasNext()
-    {
-        if ( iter1 != null )
-        {
-            if ( iter1.hasNext() ) return true ;
+    public boolean hasNext() {
+        if (iter1 != null) {
+            if (iter1.hasNext())
+                return true;
             // Iter1 ends
-            iter1 = null ;
+            iter1 = null;
         }
-        
-        if ( iter2 != null )
-        {
-            if ( iter2.hasNext() ) return true ;
+
+        if (iter2 != null) {
+            if (iter2.hasNext())
+                return true;
             // Iter2 ends
-            iter2 = null ;
+            iter2 = null;
         }
-        return false ; 
+        return false;
     }
 
     @Override
-    public T next()
-    {
-        if ( ! hasNext() )
-            throw new NoSuchElementException("Iterator2.next") ;
-        if ( iter1 != null )
-        {
-            removeFrom = iter1 ;
+    public T next() {
+        if (!hasNext())
+            throw new NoSuchElementException("Iterator2.next");
+        if (iter1 != null) {
+            removeFrom = iter1;
             return iter1.next();
         }
-        if ( iter2 != null )
-        {
-            removeFrom = iter2 ;
+        if (iter2 != null) {
+            removeFrom = iter2;
             return iter2.next();
         }
-        throw new Error("Iterator2.next") ;
+        throw new Error("Iterator2.next");
     }
 
     @Override
-    public void remove()
-    {
-        if ( null == removeFrom )
-            throw new IllegalStateException("no calls to next() since last call to remove()")
;
-        
-        removeFrom.remove() ;
-        removeFrom = null ;
+    public void remove() {
+        if (null == removeFrom)
+            throw new IllegalStateException("no calls to next() since last call to remove()");
+
+        removeFrom.remove();
+        removeFrom = null;
     }
 
     @Override
-    public Iterator<T> iterator()
-    {
-        return this ;
+    public Iterator<T> iterator() {
+        return this;
     }
 }

http://git-wip-us.apache.org/repos/asf/jena/blob/e4f6d625/jena-arq/src/main/java/org/apache/jena/atlas/lib/Map2.java
----------------------------------------------------------------------
diff --git a/jena-arq/src/main/java/org/apache/jena/atlas/lib/Map2.java b/jena-arq/src/main/java/org/apache/jena/atlas/lib/Map2.java
index 75697e0..3e3a254 100644
--- a/jena-arq/src/main/java/org/apache/jena/atlas/lib/Map2.java
+++ b/jena-arq/src/main/java/org/apache/jena/atlas/lib/Map2.java
@@ -81,7 +81,7 @@ public class Map2<K, V> implements Iterable<K>
         Iter<K> iter1 = Iter.iter(map1.keySet().iterator()) ;
         if ( map2 == null )
             return iter1 ; 
-        return iter1.append(map2.iterator()) ;
+        return iter1.concat(Iter.iter(map2.iterator())) ;
     }
     
     public boolean isEmpty()

http://git-wip-us.apache.org/repos/asf/jena/blob/e4f6d625/jena-arq/src/test/java/org/apache/jena/atlas/iterator/TestIter.java
----------------------------------------------------------------------
diff --git a/jena-arq/src/test/java/org/apache/jena/atlas/iterator/TestIter.java b/jena-arq/src/test/java/org/apache/jena/atlas/iterator/TestIter.java
index f611c03..a8b9937 100644
--- a/jena-arq/src/test/java/org/apache/jena/atlas/iterator/TestIter.java
+++ b/jena-arq/src/test/java/org/apache/jena/atlas/iterator/TestIter.java
@@ -37,29 +37,29 @@ public class TestIter
     List<String> data3 = Arrays.asList(null, "x", null, null, null, "y", "z", null);
  
     @Test
-    public void append_1()
+    public void concat_1()
     {
-        Iterator<String> iter = Iter.append(data1, data0) ;
+        Iterator<String> iter = Iter.concat(data1, data0) ;
         test(iter, "a") ;
     }
         
         
     @Test
-    public void append_2()
+    public void concat_2()
     {
-        Iterator<String> iter = Iter.append(data0, data1) ;
+        Iterator<String> iter = Iter.concat(data0, data1) ;
         test(iter, "a") ;
     }
         
     @Test
-    public void append_3()
+    public void concat_3()
     {
-        Iterator<String> iter = Iter.append(data1, data2) ;
+        Iterator<String> iter = Iter.concat(data1, data2) ;
         test(iter, "a", "x", "y", "z") ;
     }
 
     @Test
-    public void append_4()
+    public void concat_4()
     {
         List<String> L = new ArrayList<>(3);
         L.add("a");
@@ -71,7 +71,7 @@ public class TestIter
         R.add("f");
         
         
-        Iterator<String> LR = Iter.append(L, R) ;
+        Iterator<String> LR = Iter.concat(L, R) ;
         
         while (LR.hasNext())
         {
@@ -89,7 +89,7 @@ public class TestIter
     }
     
     @Test
-    public void append_5()
+    public void concat_5()
     {
         List<String> L = new ArrayList<>(3);
         L.add("a");
@@ -101,7 +101,7 @@ public class TestIter
         R.add("f");
         
         
-        Iterator<String> LR = Iter.append(L, R) ;
+        Iterator<String> LR = Iter.concat(L, R) ;
         
         while (LR.hasNext())
         {
@@ -119,7 +119,7 @@ public class TestIter
     }
     
     @Test
-    public void append_6()
+    public void concat_6()
     {
         List<String> L = new ArrayList<>(3);
         L.add("a");
@@ -131,7 +131,7 @@ public class TestIter
         R.add("f");
         
         
-        Iterator<String> LR = Iter.append(L, R) ;
+        Iterator<String> LR = Iter.concat(L, R) ;
         
         while (LR.hasNext())
         {

http://git-wip-us.apache.org/repos/asf/jena/blob/e4f6d625/jena-arq/src/test/java/org/apache/jena/atlas/iterator/TestIteratorPeek.java
----------------------------------------------------------------------
diff --git a/jena-arq/src/test/java/org/apache/jena/atlas/iterator/TestIteratorPeek.java b/jena-arq/src/test/java/org/apache/jena/atlas/iterator/TestIteratorPeek.java
index 33f0b70..59b3dee 100644
--- a/jena-arq/src/test/java/org/apache/jena/atlas/iterator/TestIteratorPeek.java
+++ b/jena-arq/src/test/java/org/apache/jena/atlas/iterator/TestIteratorPeek.java
@@ -45,7 +45,7 @@ public class TestIteratorPeek extends BaseTest
     @Test public void iter_01() 
     {
         Iter<String> iter = Iter.iter(data2) ;
-        iter = iter.append(data2.iterator()) ;
+        iter = iter.concat(data2.iterator()) ;
         test(iter, "x", "y", "z", "x", "y", "z") ;
     }
     


Mime
View raw message