marmotta-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sschaff...@apache.org
Subject git commit: fixing some test errors, perparing for release
Date Fri, 13 Sep 2013 10:23:01 GMT
Updated Branches:
  refs/heads/develop ec26977c9 -> c7563c0d7


fixing some test errors, perparing for release


Project: http://git-wip-us.apache.org/repos/asf/incubator-marmotta/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-marmotta/commit/c7563c0d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-marmotta/tree/c7563c0d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-marmotta/diff/c7563c0d

Branch: refs/heads/develop
Commit: c7563c0d7d1030dc60b5e7682ac7cae0adf34331
Parents: ec26977
Author: Sebastian Schaffert <sschaffert@apache.org>
Authored: Fri Sep 13 12:22:55 2013 +0200
Committer: Sebastian Schaffert <sschaffert@apache.org>
Committed: Fri Sep 13 12:22:55 2013 +0200

----------------------------------------------------------------------
 .../concurrent/ConcurrentFacadingTest.java      |   3 +
 .../kiwi/reasoner/engine/ReasoningEngine.java   |  42 ++++++--
 .../test/engine/ReasoningEngineTest.java        |   2 +-
 .../reasoner/test/sail/ReasoningSailTest.java   |   1 +
 .../KiWiRDFSchemaRepositoryConnectionTest.java  |   2 +-
 .../kiwi/persistence/KiWiConnection.java        | 108 +++++++++++--------
 .../kiwi/persistence/h2/statements.properties   |   4 +-
 .../kiwi/test/sesame/KiWiLocaleTest.java        |   5 +-
 8 files changed, 109 insertions(+), 58 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-marmotta/blob/c7563c0d/commons/sesame-tools-facading/src/test/java/org/apache/marmotta/commons/sesame/facading/concurrent/ConcurrentFacadingTest.java
----------------------------------------------------------------------
diff --git a/commons/sesame-tools-facading/src/test/java/org/apache/marmotta/commons/sesame/facading/concurrent/ConcurrentFacadingTest.java
b/commons/sesame-tools-facading/src/test/java/org/apache/marmotta/commons/sesame/facading/concurrent/ConcurrentFacadingTest.java
index 504699d..03cdc09 100644
--- a/commons/sesame-tools-facading/src/test/java/org/apache/marmotta/commons/sesame/facading/concurrent/ConcurrentFacadingTest.java
+++ b/commons/sesame-tools-facading/src/test/java/org/apache/marmotta/commons/sesame/facading/concurrent/ConcurrentFacadingTest.java
@@ -1,5 +1,6 @@
 package org.apache.marmotta.commons.sesame.facading.concurrent;
 
+import java.util.ConcurrentModificationException;
 import java.util.UUID;
 
 import org.apache.marmotta.commons.sesame.facading.AbstractFacadingTest;
@@ -169,6 +170,8 @@ public class ConcurrentFacadingTest extends AbstractFacadingTest {
                     subCon_2.close();
                 }
             }
+        } catch (ConcurrentModificationException ex) {
+            // do nothing, H2 locking
         } finally {
             if (mainCon.isActive()) mainCon.rollback();
             mainCon.close();

http://git-wip-us.apache.org/repos/asf/incubator-marmotta/blob/c7563c0d/libraries/kiwi/kiwi-reasoner/src/main/java/org/apache/marmotta/kiwi/reasoner/engine/ReasoningEngine.java
----------------------------------------------------------------------
diff --git a/libraries/kiwi/kiwi-reasoner/src/main/java/org/apache/marmotta/kiwi/reasoner/engine/ReasoningEngine.java
b/libraries/kiwi/kiwi-reasoner/src/main/java/org/apache/marmotta/kiwi/reasoner/engine/ReasoningEngine.java
index 18e0607..47891c6 100644
--- a/libraries/kiwi/kiwi-reasoner/src/main/java/org/apache/marmotta/kiwi/reasoner/engine/ReasoningEngine.java
+++ b/libraries/kiwi/kiwi-reasoner/src/main/java/org/apache/marmotta/kiwi/reasoner/engine/ReasoningEngine.java
@@ -121,6 +121,7 @@ public class ReasoningEngine implements TransactionListener {
      */
     private static long taskCounter = 0;
 
+    private boolean isshutdown = false;
 
     /**
      * The worker thread for the reasoner.
@@ -894,17 +895,44 @@ public class ReasoningEngine implements TransactionListener {
     }
 
     public void shutdown() {
-        log.info("shutting down reasoning service ...");
+        shutdown(false);
+    }
 
-        for(int i = 0; i<20 && isRunning(); i++) {
-            log.warn("reasoner not yet finished, waiting for 1 seconds (try={})", i+1);
-            try {
-                Thread.sleep(1000);
-            } catch (InterruptedException e) {
+    public void shutdown(boolean force) {
+        if(isshutdown)
+            return;
+
+        if(force) {
+            log.warn("forced shutdown of reasoning service initiated, state will be inconsistent
...");
+
+            reasonerThread.shutdown();
+
+            for(int i = 0; i<3 && isRunning(); i++) {
+                log.warn("reasoner not yet finished, waiting for 1 seconds (try={})", i+1);
+                try {
+                    Thread.sleep(1000);
+                } catch (InterruptedException e) {
+                }
             }
+
+            // yes, I know it is unsafe; it is only used when forcefully shutting down on
test ends before the database is deleted...
+            reasonerThread.stop();
+
+        } else {
+            log.info("graceful shutdown of reasoning service initiated ...");
+
+            for(int i = 0; i<20 && isRunning(); i++) {
+                log.warn("reasoner not yet finished, waiting for 1 seconds (try={})", i+1);
+                try {
+                    Thread.sleep(1000);
+                } catch (InterruptedException e) {
+                }
+            }
+
+            reasonerThread.shutdown();
         }
 
-        reasonerThread.shutdown();
+        isshutdown = true;
     }
 
     /**

http://git-wip-us.apache.org/repos/asf/incubator-marmotta/blob/c7563c0d/libraries/kiwi/kiwi-reasoner/src/test/java/org/apache/marmotta/kiwi/reasoner/test/engine/ReasoningEngineTest.java
----------------------------------------------------------------------
diff --git a/libraries/kiwi/kiwi-reasoner/src/test/java/org/apache/marmotta/kiwi/reasoner/test/engine/ReasoningEngineTest.java
b/libraries/kiwi/kiwi-reasoner/src/test/java/org/apache/marmotta/kiwi/reasoner/test/engine/ReasoningEngineTest.java
index 75b7cad..fb2c497 100644
--- a/libraries/kiwi/kiwi-reasoner/src/test/java/org/apache/marmotta/kiwi/reasoner/test/engine/ReasoningEngineTest.java
+++ b/libraries/kiwi/kiwi-reasoner/src/test/java/org/apache/marmotta/kiwi/reasoner/test/engine/ReasoningEngineTest.java
@@ -129,7 +129,7 @@ public class ReasoningEngineTest {
 
     @After
     public void dropDatabase() throws Exception {
-        engine.shutdown();
+        engine.shutdown(true);
 
         rpersistence.dropDatabase();
         persistence.dropDatabase();

http://git-wip-us.apache.org/repos/asf/incubator-marmotta/blob/c7563c0d/libraries/kiwi/kiwi-reasoner/src/test/java/org/apache/marmotta/kiwi/reasoner/test/sail/ReasoningSailTest.java
----------------------------------------------------------------------
diff --git a/libraries/kiwi/kiwi-reasoner/src/test/java/org/apache/marmotta/kiwi/reasoner/test/sail/ReasoningSailTest.java
b/libraries/kiwi/kiwi-reasoner/src/test/java/org/apache/marmotta/kiwi/reasoner/test/sail/ReasoningSailTest.java
index 15f4d8f..64da3c1 100644
--- a/libraries/kiwi/kiwi-reasoner/src/test/java/org/apache/marmotta/kiwi/reasoner/test/sail/ReasoningSailTest.java
+++ b/libraries/kiwi/kiwi-reasoner/src/test/java/org/apache/marmotta/kiwi/reasoner/test/sail/ReasoningSailTest.java
@@ -88,6 +88,7 @@ public class ReasoningSailTest {
 
     @After
     public void dropDatabase() throws Exception {
+        rsail.getEngine().shutdown(true);
         rsail.getPersistence().dropDatabase();
         store.getPersistence().dropDatabase();
         repository.shutDown();

http://git-wip-us.apache.org/repos/asf/incubator-marmotta/blob/c7563c0d/libraries/kiwi/kiwi-reasoner/src/test/java/org/apache/marmotta/kiwi/reasoner/test/sesame/KiWiRDFSchemaRepositoryConnectionTest.java
----------------------------------------------------------------------
diff --git a/libraries/kiwi/kiwi-reasoner/src/test/java/org/apache/marmotta/kiwi/reasoner/test/sesame/KiWiRDFSchemaRepositoryConnectionTest.java
b/libraries/kiwi/kiwi-reasoner/src/test/java/org/apache/marmotta/kiwi/reasoner/test/sesame/KiWiRDFSchemaRepositoryConnectionTest.java
index 8d959ec..90bf0f9 100644
--- a/libraries/kiwi/kiwi-reasoner/src/test/java/org/apache/marmotta/kiwi/reasoner/test/sesame/KiWiRDFSchemaRepositoryConnectionTest.java
+++ b/libraries/kiwi/kiwi-reasoner/src/test/java/org/apache/marmotta/kiwi/reasoner/test/sesame/KiWiRDFSchemaRepositoryConnectionTest.java
@@ -80,7 +80,7 @@ public class KiWiRDFSchemaRepositoryConnectionTest extends RDFSchemaRepositoryCo
         Sail wsail = new SailWrapper(rsail) {
             @Override
             public void shutDown() throws SailException {
-                rsail.getEngine().shutdown();
+                rsail.getEngine().shutdown(true);
 
                 try {
                     rsail.getPersistence().dropDatabase();

http://git-wip-us.apache.org/repos/asf/incubator-marmotta/blob/c7563c0d/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/persistence/KiWiConnection.java
----------------------------------------------------------------------
diff --git a/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/persistence/KiWiConnection.java
b/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/persistence/KiWiConnection.java
index 119256c..9c7de67 100644
--- a/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/persistence/KiWiConnection.java
+++ b/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/persistence/KiWiConnection.java
@@ -28,7 +28,6 @@ import com.google.common.base.Preconditions;
 import net.sf.ehcache.Cache;
 import net.sf.ehcache.Element;
 
-import org.apache.commons.lang3.LocaleUtils;
 import org.apache.marmotta.kiwi.caching.KiWiCacheManager;
 import org.apache.marmotta.kiwi.config.KiWiConfiguration;
 import org.apache.marmotta.kiwi.model.caching.TripleTable;
@@ -46,7 +45,6 @@ import org.slf4j.LoggerFactory;
 import java.sql.*;
 import java.util.*;
 import java.util.Date;
-import java.util.concurrent.Executor;
 import java.util.concurrent.locks.ReentrantLock;
 
 /**
@@ -1062,7 +1060,7 @@ public class KiWiConnection {
      * @throws NullPointerException in case the subject, predicate, object or context have
not been persisted
      * @return true in case the update added a new triple to the database, false in case
the triple already existed
      */
-    public synchronized boolean storeTriple(KiWiTriple triple) throws SQLException {
+    public synchronized boolean storeTriple(final KiWiTriple triple) throws SQLException
{
         // mutual exclusion: prevent parallel adding and removing of the same triple
         synchronized (triple) {
 
@@ -1104,30 +1102,39 @@ public class KiWiConnection {
 
 
                     try {
-                        PreparedStatement insertTriple = getPreparedStatement("store.triple");
-                        insertTriple.setLong(1,triple.getId());
-                        insertTriple.setLong(2,triple.getSubject().getId());
-                        insertTriple.setLong(3,triple.getPredicate().getId());
-                        insertTriple.setLong(4,triple.getObject().getId());
-                        if(triple.getContext() != null) {
-                            insertTriple.setLong(5,triple.getContext().getId());
-                        } else {
-                            insertTriple.setNull(5, Types.BIGINT);
-                        }
-                        insertTriple.setBoolean(6,triple.isInferred());
-                        insertTriple.setTimestamp(7, new Timestamp(triple.getCreated().getTime()));
-                        int count = insertTriple.executeUpdate();
+                        RetryExecution<Boolean> execution = new RetryExecution<>("STORE");
+                        execution.setUseSavepoint(true);
+                        execution.execute(connection, new RetryCommand<Boolean>() {
+                            @Override
+                            public Boolean run() throws SQLException {
+                                PreparedStatement insertTriple = getPreparedStatement("store.triple");
+                                insertTriple.setLong(1,triple.getId());
+                                insertTriple.setLong(2,triple.getSubject().getId());
+                                insertTriple.setLong(3,triple.getPredicate().getId());
+                                insertTriple.setLong(4,triple.getObject().getId());
+                                if(triple.getContext() != null) {
+                                    insertTriple.setLong(5,triple.getContext().getId());
+                                } else {
+                                    insertTriple.setNull(5, Types.BIGINT);
+                                }
+                                insertTriple.setBoolean(6,triple.isInferred());
+                                insertTriple.setTimestamp(7, new Timestamp(triple.getCreated().getTime()));
+                                int count = insertTriple.executeUpdate();
 
-                        cacheTriple(triple);
+                                cacheTriple(triple);
+
+                                return count > 0;
+                            }
+                        });
+
+                        return !hasId;
 
-                        return count > 0;
                     } catch(SQLException ex) {
                         if("HYT00".equals(ex.getSQLState())) { // H2 table locking timeout
                             throw new ConcurrentModificationException("the same triple was
modified in concurrent transactions (triple="+triple+")");
+                        } else {
+                            throw ex;
                         }
-                        // this is an ugly hack to catch duplicate key errors in some databases
(H2)
-                        // better option could be http://stackoverflow.com/questions/6736518/h2-java-insert-ignore-allow-exception
-                        return false;
                     }
                 }
             }
@@ -1192,9 +1199,9 @@ public class KiWiConnection {
 
         RetryExecution execution = new RetryExecution("DELETE");
         execution.setUseSavepoint(true);
-        execution.execute(connection, new RetryCommand() {
+        execution.execute(connection, new RetryCommand<Void>() {
             @Override
-            public void run() throws SQLException {
+            public Void run() throws SQLException {
                 // mutual exclusion: prevent parallel adding and removing of the same triple
                 synchronized (triple) {
 
@@ -1202,19 +1209,19 @@ public class KiWiConnection {
                     triple.setDeleted(true);
                     triple.setDeletedAt(new Date());
 
-                    if(triple.getId() == null) {
+                    if (triple.getId() == null) {
                         log.warn("attempting to remove non-persistent triple: {}", triple);
                         removeCachedTriple(triple);
                     } else {
-                        if(batchCommit) {
+                        if (batchCommit) {
                             // need to remove from triple batch and from database
                             commitLock.lock();
                             try {
-                                if(tripleBatch == null || !tripleBatch.remove(triple)) {
+                                if (tripleBatch == null || !tripleBatch.remove(triple)) {
 
                                     PreparedStatement deleteTriple = getPreparedStatement("delete.triple");
                                     synchronized (deleteTriple) {
-                                        deleteTriple.setLong(1,triple.getId());
+                                        deleteTriple.setLong(1, triple.getId());
                                         deleteTriple.executeUpdate();
                                     }
                                     deletedStatementsLog.add(triple.getId());
@@ -1227,7 +1234,7 @@ public class KiWiConnection {
 
                             PreparedStatement deleteTriple = getPreparedStatement("delete.triple");
                             synchronized (deleteTriple) {
-                                deleteTriple.setLong(1,triple.getId());
+                                deleteTriple.setLong(1, triple.getId());
                                 deleteTriple.executeUpdate();
                             }
                             deletedStatementsLog.add(triple.getId());
@@ -1237,6 +1244,8 @@ public class KiWiConnection {
                         removeCachedTriple(triple);
                     }
                 }
+
+                return null;
             }
         });
 
@@ -1789,7 +1798,7 @@ public class KiWiConnection {
 
     private void cacheNode(KiWiNode node) {
         if(node.getId() != null) {
-            nodeCache.put(new Element(node.getId(),node));
+            nodeCache.put(new Element(node.getId(), node));
         }
         if(node instanceof KiWiUriResource) {
             uriCache.put(new Element(node.stringValue(), node));
@@ -1976,9 +1985,9 @@ public class KiWiConnection {
         numberOfCommits++;
 
         RetryExecution execution = new RetryExecution("COMMIT");
-        execution.execute(connection, new RetryCommand() {
+        execution.execute(connection, new RetryCommand<Void>() {
             @Override
-            public void run() throws SQLException {
+            public Void run() throws SQLException {
                 if(persistence.getConfiguration().isCommitSequencesOnCommit() || numberOfCommits
% 100 == 0) {
                     commitMemorySequences();
                 }
@@ -1994,6 +2003,8 @@ public class KiWiConnection {
                 if(connection != null) {
                     connection.commit();
                 }
+
+                return null;
             }
         });
     }
@@ -2135,9 +2146,9 @@ public class KiWiConnection {
 
                 RetryExecution execution = new RetryExecution("FLUSH BATCH");
                 execution.setUseSavepoint(true);
-                execution.execute(connection, new RetryCommand() {
+                execution.execute(connection, new RetryCommand<Void>() {
                     @Override
-                    public void run() throws SQLException {
+                    public Void run() throws SQLException {
                         PreparedStatement insertTriple = getPreparedStatement("store.triple");
                         insertTriple.clearParameters();
                         insertTriple.clearBatch();
@@ -2177,6 +2188,7 @@ public class KiWiConnection {
 
                         tripleBatch.clear();
 
+                        return null;
                     }
                 });
 
@@ -2189,9 +2201,9 @@ public class KiWiConnection {
     }
 
 
-    protected static interface RetryCommand {
+    protected static interface RetryCommand<T> {
 
-        public void run() throws SQLException;
+        public T run() throws SQLException;
     }
 
     /**
@@ -2199,7 +2211,7 @@ public class KiWiConnection {
      * and should be retried several times before giving up completely.
      *
      */
-    protected static class RetryExecution  {
+    protected static class RetryExecution<T>  {
 
         // counter for current number of retries
         private int retries = 0;
@@ -2260,36 +2272,40 @@ public class KiWiConnection {
             return sqlStates;
         }
 
-        public void execute(Connection connection, RetryCommand command) throws SQLException
{
+        public T execute(Connection connection, RetryCommand<T> command) throws SQLException
{
             Savepoint savepoint = null;
             if(useSavepoint) {
                 savepoint = connection.setSavepoint();
             }
             try {
-                command.run();
+                T result = command.run();
 
                 if(useSavepoint && savepoint != null) {
                     connection.releaseSavepoint(savepoint);
                 }
+
+                return result;
             } catch (SQLException ex) {
                 if(retries < maxRetries && (sqlStates.size() == 0 || sqlStates.contains(ex.getSQLState())))
{
                     if(useSavepoint && savepoint != null) {
                         connection.rollback(savepoint);
                     }
-                    log.warn("{}: temporary conflict, retrying in {} ms ... (thread={}, retry={})",
name, retryInterval, Thread.currentThread().getName(), retries);
+                    Random rnd = new Random();
+                    long sleep = retryInterval - 250 + rnd.nextInt(500);
+                    log.warn("{}: temporary conflict, retrying in {} ms ... (thread={}, retry={})",
name, sleep, Thread.currentThread().getName(), retries);
                     try {
-                        Thread.sleep(retryInterval);
+                        Thread.sleep(sleep);
                     } catch (InterruptedException e) {}
                     retries++;
-                    execute(connection, command);
+                    T result = execute(connection, command);
                     retries--;
+
+                    return result;
                 } else {
-                    log.error("{}: temporary conflict could not be solved!", name);
+                    log.error("{}: temporary conflict could not be solved! (error: {})",
name, ex.getMessage());
 
-                    System.err.println("main exception:");
-                    ex.printStackTrace();
-                    System.err.println("next exception:");
-                    ex.getNextException().printStackTrace();
+                    log.debug("main exception:",ex);
+                    log.debug("next exception:",ex.getNextException());
                     throw ex;
                 }
             }

http://git-wip-us.apache.org/repos/asf/incubator-marmotta/blob/c7563c0d/libraries/kiwi/kiwi-triplestore/src/main/resources/org/apache/marmotta/kiwi/persistence/h2/statements.properties
----------------------------------------------------------------------
diff --git a/libraries/kiwi/kiwi-triplestore/src/main/resources/org/apache/marmotta/kiwi/persistence/h2/statements.properties
b/libraries/kiwi/kiwi-triplestore/src/main/resources/org/apache/marmotta/kiwi/persistence/h2/statements.properties
index 130e3e7..092cfcb 100644
--- a/libraries/kiwi/kiwi-triplestore/src/main/resources/org/apache/marmotta/kiwi/persistence/h2/statements.properties
+++ b/libraries/kiwi/kiwi-triplestore/src/main/resources/org/apache/marmotta/kiwi/persistence/h2/statements.properties
@@ -60,8 +60,8 @@ store.tliteral       = INSERT INTO nodes (id,ntype,svalue,tvalue,ltype,createdAt
 
 store.namespace      = INSERT INTO namespaces (id,prefix,uri,createdAt) VALUES (?,?,?,?)
 
-store.triple         = INSERT INTO triples (id,subject,predicate,object,context,inferred,createdAt)
VALUES (?,?,?,?,?,?,?)
-#store.triple         = MERGE INTO triples (id,subject,predicate,object,context,inferred,createdAt)
KEY(id) VALUES (?,?,?,?,?,?,?)
+#store.triple         = INSERT INTO triples (id,subject,predicate,object,context,inferred,createdAt)
VALUES (?,?,?,?,?,?,?)
+store.triple         = MERGE INTO triples (id,subject,predicate,object,context,inferred,createdAt)
KEY(id) VALUES (?,?,?,?,?,?,?)
 load.triple          = SELECT id FROM triples WHERE subject = ? AND predicate = ? AND object
= ? AND context = ? AND deleted = false AND inferred = true
 
 

http://git-wip-us.apache.org/repos/asf/incubator-marmotta/blob/c7563c0d/libraries/kiwi/kiwi-triplestore/src/test/java/org/apache/marmotta/kiwi/test/sesame/KiWiLocaleTest.java
----------------------------------------------------------------------
diff --git a/libraries/kiwi/kiwi-triplestore/src/test/java/org/apache/marmotta/kiwi/test/sesame/KiWiLocaleTest.java
b/libraries/kiwi/kiwi-triplestore/src/test/java/org/apache/marmotta/kiwi/test/sesame/KiWiLocaleTest.java
index cbdbee4..487bfde 100644
--- a/libraries/kiwi/kiwi-triplestore/src/test/java/org/apache/marmotta/kiwi/test/sesame/KiWiLocaleTest.java
+++ b/libraries/kiwi/kiwi-triplestore/src/test/java/org/apache/marmotta/kiwi/test/sesame/KiWiLocaleTest.java
@@ -30,6 +30,8 @@ import org.openrdf.model.Literal;
 import org.openrdf.sail.SailConnection;
 import org.openrdf.sail.SailException;
 
+import java.sql.SQLException;
+
 /**
  * Tests for testing locales against the KiWi Triple store 
  * (and its implementation of the ValueFactory)
@@ -55,7 +57,8 @@ public class KiWiLocaleTest  {
     }
     
     @After
-    public void shutdown() throws SailException {
+    public void shutdown() throws SailException, SQLException {
+        store.getPersistence().dropDatabase();
     	store.shutDown();
     	store = null;
     	vf = null;


Mime
View raw message