Mailing-List: contact commits-help@marmotta.apache.org; run by ezmlm
Precedence: bulk
Reply-To: dev@marmotta.apache.org
Content-Type: text/plain; charset="us-ascii"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
From: sschaffert@apache.org
To: commits@marmotta.apache.org
Message-Id: <a0c2b23d0fe54e0f8bc98d5ef434bbc6@git.apache.org>
Subject: git commit: further code cleanups and optimizations
Date: Wed,  5 Feb 2014 12:54:22 +0000 (UTC)

Updated Branches:
  refs/heads/develop af130d6da -> 9ea6db0af


further code cleanups and optimizations


Project: http://git-wip-us.apache.org/repos/asf/marmotta/repo
Commit: http://git-wip-us.apache.org/repos/asf/marmotta/commit/9ea6db0a
Tree: http://git-wip-us.apache.org/repos/asf/marmotta/tree/9ea6db0a
Diff: http://git-wip-us.apache.org/repos/asf/marmotta/diff/9ea6db0a

Branch: refs/heads/develop
Commit: 9ea6db0af9c783fc45595d1351dfd65c355f870c
Parents: af130d6
Author: Sebastian Schaffert <sschaffert@apache.org>
Authored: Wed Feb 5 13:54:15 2014 +0100
Committer: Sebastian Schaffert <sschaffert@apache.org>
Committed: Wed Feb 5 13:54:15 2014 +0100

----------------------------------------------------------------------
 .../kiwi/persistence/KiWiConnection.java        | 91 +++++++++++++-------
 1 file changed, 59 insertions(+), 32 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/marmotta/blob/9ea6db0a/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/persistence/KiWiConnection.java
----------------------------------------------------------------------
diff --git a/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/persistence/KiWiConnection.java b/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/persistence/KiWiConnection.java
index 500b324..6a067dc 100644
--- a/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/persistence/KiWiConnection.java
+++ b/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/persistence/KiWiConnection.java
@@ -138,6 +138,8 @@ public class KiWiConnection implements AutoCloseable {
 
     private long transactionId;
 
+    private int QUERY_BATCH_SIZE = 1024;
+
     public KiWiConnection(KiWiPersistence persistence, KiWiDialect dialect, KiWiCacheManager cacheManager) throws SQLException {
         this.cacheManager = cacheManager;
         this.dialect      = dialect;
@@ -469,35 +471,56 @@ public class KiWiConnection implements AutoCloseable {
         }
 
         if(toFetch.size() > 0) {
-            PreparedStatement query = getPreparedStatement("load.nodes_by_ids", toFetch.size());
-            synchronized (query) {
+            // declare variables before to optimize stack allocation
+            int position = 0;
+            int nextBatchSize;
+            PreparedStatement query;
+            KiWiNode node;
 
-                for(int i=0; i<toFetch.size(); i++) {
-                    query.setLong(i+1, toFetch.get(i));
-                }
-                query.setMaxRows(toFetch.size());
+            while(position < toFetch.size()) {
+                nextBatchSize = computeBatchSize(position, toFetch.size());
 
-                // run the database query and if it yields a result, construct a new node; the method call will take care of
-                // caching the constructed node for future calls
-                ResultSet rows = query.executeQuery();
-                try {
-                    while(rows.next()) {
-                        KiWiNode n = constructNodeFromDatabase(rows);
-                        for(int i=0; i<ids.length; i++) {
-                            if(ids[i].longValue() == n.getId()) {
-                                result[i] = n;
+                query = getPreparedStatement("load.nodes_by_ids", nextBatchSize);
+                synchronized (query) {
+
+                    for(int i=0; i<nextBatchSize; i++) {
+                        query.setLong(i+1, toFetch.get(position + i));
+                    }
+                    query.setMaxRows(nextBatchSize);
+
+                    // run the database query and if it yields a result, construct a new node; the method call will take care of
+                    // caching the constructed node for future calls
+                    ResultSet rows = query.executeQuery();
+                    try {
+                        while(rows.next()) {
+                            node = constructNodeFromDatabase(rows);
+                            for(int i=0; i<ids.length; i++) {
+                                if(ids[i].longValue() == node.getId()) {
+                                    result[i] = node;
+                                }
                             }
                         }
+                    } finally {
+                        rows.close();
                     }
-                } finally {
-                    rows.close();
+
+                    position += nextBatchSize;
                 }
             }
 
+
         }
         return result;
     }
 
+    private int computeBatchSize(int position, int length) {
+        int batchSize = QUERY_BATCH_SIZE;
+        while(length - position < batchSize) {
+            batchSize = batchSize >> 1;
+        }
+        return batchSize;
+    }
+
     public KiWiTriple loadTripleById(Long id) throws SQLException {
 
         // look in cache
@@ -1577,7 +1600,7 @@ public class KiWiConnection implements AutoCloseable {
 
             private void fetchBatch() throws SQLException {
                 if(batch == null || batch.size() <= batchPosition) {
-                    batch = constructTriplesFromDatabase(result, 100);
+                    batch = constructTriplesFromDatabase(result, QUERY_BATCH_SIZE);
                     batchPosition = 0;
                 }
             }
@@ -1803,6 +1826,10 @@ public class KiWiConnection implements AutoCloseable {
     protected List<KiWiTriple> constructTriplesFromDatabase(ResultSet row, int maxPrefetch) throws SQLException {
         int count = 0;
 
+        // declare variables to optimize stack allocation
+        KiWiTriple triple;
+        long id;
+
         List<KiWiTriple> result = new ArrayList<>();
         Map<Long,Long[]> tripleIds  = new HashMap<>();
         Set<Long> nodeIds   = new HashSet<>();
@@ -1816,16 +1843,16 @@ public class KiWiConnection implements AutoCloseable {
             // columns: id,subject,predicate,object,context,deleted,inferred,creator,createdAt,deletedAt
             //          1 ,2      ,3        ,4     ,5      ,6      ,7       ,8      ,9        ,10
 
-            Long id = row.getLong(1);
+            id = row.getLong(1);
 
-            KiWiTriple cached = tripleCache.get(id);
+            triple = tripleCache.get(id);
 
             // lookup element in cache first, so we can avoid reconstructing it if it is already there
-            if(cached != null) {
-                result.add(cached);
+            if(triple != null) {
+                result.add(triple);
             } else {
 
-                KiWiTriple triple = new KiWiTriple();
+                triple = new KiWiTriple();
                 triple.setId(id);
 
                 // collect node ids for batch retrieval
@@ -1866,25 +1893,25 @@ public class KiWiConnection implements AutoCloseable {
             nodeMap.put(nodes[i].getId(), nodes[i]);
         }
 
-        for(KiWiTriple triple : result) {
-            if(tripleIds.containsKey(triple.getId())) {
+        for(KiWiTriple t : result) {
+            if(tripleIds.containsKey(t.getId())) {
                 // need to set subject, predicate, object, context and creator
-                Long[] ids = tripleIds.get(triple.getId());
-                triple.setSubject((KiWiResource) nodeMap.get(ids[0]));
-                triple.setPredicate((KiWiUriResource) nodeMap.get(ids[1]));
-                triple.setObject(nodeMap.get(ids[2]));
+                Long[] ids = tripleIds.get(t.getId());
+                t.setSubject((KiWiResource) nodeMap.get(ids[0]));
+                t.setPredicate((KiWiUriResource) nodeMap.get(ids[1]));
+                t.setObject(nodeMap.get(ids[2]));
 
                 if(ids[3] != 0) {
-                    triple.setContext((KiWiResource) nodeMap.get(ids[3]));
+                    t.setContext((KiWiResource) nodeMap.get(ids[3]));
                 }
 
                 if(ids[4] != 0) {
-                    triple.setCreator((KiWiResource) nodeMap.get(ids[4]));
+                    t.setCreator((KiWiResource) nodeMap.get(ids[4]));
                 }
 
             }
 
-            cacheTriple(triple);
+            cacheTriple(t);
         }