marmotta-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sschaff...@apache.org
Subject [1/2] git commit: added some more safeguards to avoid database inconsistencies
Date Mon, 22 Jul 2013 13:46:28 GMT
Updated Branches:
  refs/heads/develop 6ae301d3f -> ecad38c5f


added some more safeguards to avoid database inconsistencies


Project: http://git-wip-us.apache.org/repos/asf/incubator-marmotta/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-marmotta/commit/524e9a0d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-marmotta/tree/524e9a0d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-marmotta/diff/524e9a0d

Branch: refs/heads/develop
Commit: 524e9a0d653b1adc70dd4df8042d146d4d295e0e
Parents: 2d70611
Author: Sebastian Schaffert <sschaffert@apache.org>
Authored: Mon Jul 22 15:46:02 2013 +0200
Committer: Sebastian Schaffert <sschaffert@apache.org>
Committed: Mon Jul 22 15:46:02 2013 +0200

----------------------------------------------------------------------
 .../kiwi/persistence/KiWiGarbageCollector.java  | 30 ++++++++++++++++++++
 .../kiwi/persistence/KiWiPersistence.java       |  4 +++
 .../apache/marmotta/kiwi/sail/KiWiStore.java    | 19 +++++++++----
 .../kiwi/persistence/h2/create_base_tables.sql  |  3 +-
 .../persistence/mysql/upgrade_base_001_002.sql  |  4 ++-
 .../persistence/pgsql/create_base_tables.sql    |  5 ++--
 .../marmotta/kiwi/test/H2ConcurrencyTest.java   |  6 +++-
 .../kiwi/test/MySQLConcurrencyTest.java         |  6 +++-
 .../kiwi/test/PostgreSQLConcurrencyTest.java    |  8 +++++-
 9 files changed, 73 insertions(+), 12 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-marmotta/blob/524e9a0d/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/persistence/KiWiGarbageCollector.java
----------------------------------------------------------------------
diff --git a/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/persistence/KiWiGarbageCollector.java
b/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/persistence/KiWiGarbageCollector.java
index ece139f..f769e37 100644
--- a/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/persistence/KiWiGarbageCollector.java
+++ b/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/persistence/KiWiGarbageCollector.java
@@ -22,6 +22,7 @@ import org.slf4j.LoggerFactory;
 
 import java.sql.Connection;
 import java.sql.PreparedStatement;
+import java.sql.ResultSet;
 import java.sql.SQLException;
 import java.util.HashSet;
 import java.util.Iterator;
@@ -98,6 +99,28 @@ public class KiWiGarbageCollector extends Thread {
         nodeTableDependencies.add(new TableDependency(tableName,columnName));
     }
 
+    protected boolean checkConsistency() throws SQLException {
+        boolean consistent = true;
+
+        String checkNodeDuplicatesQuery = "SELECT svalue, count(id) FROM nodes WHERE ntype='uri'
group by svalue having count(id) > 1";
+
+        try(Connection con = persistence.getJDBCConnection()) {
+            PreparedStatement checkNodeDuplicatesStatement = con.prepareStatement(checkNodeDuplicatesQuery);
+
+            ResultSet result = checkNodeDuplicatesStatement.executeQuery();
+            if(result.next()) {
+                log.warn("DATABASE INCONSISTENCY: duplicate node entries found, manual repair
required!");
+                do {
+                    log.warn(" - inconsistent resource: {}", result.getString("svalue"));
+                } while(result.next());
+
+                consistent = false;
+            }
+
+        }
+
+        return consistent;
+    }
 
     protected int garbageCollect() throws SQLException {
         round++;
@@ -172,6 +195,13 @@ public class KiWiGarbageCollector extends Thread {
             while(!shutdown) {
                 // don't run immediately on startup
                 if(started) {
+                    log.info("running database consistency checks ...");
+                    try {
+                        checkConsistency();
+                    } catch (SQLException e) {
+                        log.error("error while executing consistency checks: {}",e.getMessage());
+                    }
+
                     log.info("running garbage collection ...");
                     try {
                         int count = garbageCollect();

http://git-wip-us.apache.org/repos/asf/incubator-marmotta/blob/524e9a0d/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/persistence/KiWiPersistence.java
----------------------------------------------------------------------
diff --git a/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/persistence/KiWiPersistence.java
b/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/persistence/KiWiPersistence.java
index 12ff73f..1f42d41 100644
--- a/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/persistence/KiWiPersistence.java
+++ b/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/persistence/KiWiPersistence.java
@@ -580,4 +580,8 @@ public class KiWiPersistence {
     public void garbageCollect() throws SQLException {
         this.garbageCollector.garbageCollect();
     }
+
+    public boolean checkConsistency() throws SQLException {
+        return garbageCollector.checkConsistency();
+    }
 }

http://git-wip-us.apache.org/repos/asf/incubator-marmotta/blob/524e9a0d/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/sail/KiWiStore.java
----------------------------------------------------------------------
diff --git a/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/sail/KiWiStore.java
b/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/sail/KiWiStore.java
index 6d0da12..927bc9a 100644
--- a/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/sail/KiWiStore.java
+++ b/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/sail/KiWiStore.java
@@ -92,19 +92,19 @@ public class KiWiStore extends NotifyingSailBase {
     protected ConcurrentMap<IntArray,Statement> tripleRegistry;
 
     public KiWiStore(KiWiPersistence persistence, String defaultContext, String inferredContext)
{
-    	this.persistence    = persistence;
-    	this.defaultContext = defaultContext;
+        this.persistence    = persistence;
+        this.defaultContext = defaultContext;
         this.nodeLock       = new ReentrantLock();
         this.tripleLock     = new ReentrantLock();
         this.inferredContext = inferredContext;
-    	
-    	tripleRegistry  = new MapMaker().weakValues().makeMap();
+
+        tripleRegistry  = new MapMaker().weakValues().makeMap();
 
     }
 
     @Deprecated
     public KiWiStore(String name, String jdbcUrl, String db_user, String db_password, KiWiDialect
dialect, String defaultContext, String inferredContext) {
-    	this(new KiWiConfiguration(name,jdbcUrl,db_user,db_password,dialect, defaultContext,
inferredContext));
+        this(new KiWiConfiguration(name,jdbcUrl,db_user,db_password,dialect, defaultContext,
inferredContext));
     }
 
     public KiWiStore(KiWiConfiguration configuration) {
@@ -223,4 +223,13 @@ public class KiWiStore extends NotifyingSailBase {
             throw new SailException("error calling garbage collector",e);
         }
     }
+
+
+    public boolean checkConsistency() throws SailException {
+        try {
+            return persistence.checkConsistency();
+        } catch (SQLException e) {
+            throw new SailException("error calling consistency check",e);
+        }
+    }
 }

http://git-wip-us.apache.org/repos/asf/incubator-marmotta/blob/524e9a0d/libraries/kiwi/kiwi-triplestore/src/main/resources/org/apache/marmotta/kiwi/persistence/h2/create_base_tables.sql
----------------------------------------------------------------------
diff --git a/libraries/kiwi/kiwi-triplestore/src/main/resources/org/apache/marmotta/kiwi/persistence/h2/create_base_tables.sql
b/libraries/kiwi/kiwi-triplestore/src/main/resources/org/apache/marmotta/kiwi/persistence/h2/create_base_tables.sql
index f6e1722..6769ca2 100644
--- a/libraries/kiwi/kiwi-triplestore/src/main/resources/org/apache/marmotta/kiwi/persistence/h2/create_base_tables.sql
+++ b/libraries/kiwi/kiwi-triplestore/src/main/resources/org/apache/marmotta/kiwi/persistence/h2/create_base_tables.sql
@@ -28,7 +28,8 @@ CREATE TABLE nodes (
   ltype     bigint     REFERENCES nodes(id),
   lang      varchar(5),
   createdAt timestamp  NOT NULL DEFAULT now(),
-  PRIMARY KEY(id)
+  PRIMARY KEY(id),
+  UNIQUE(ntype,svalue)
 );
 
 CREATE TABLE triples (

http://git-wip-us.apache.org/repos/asf/incubator-marmotta/blob/524e9a0d/libraries/kiwi/kiwi-triplestore/src/main/resources/org/apache/marmotta/kiwi/persistence/mysql/upgrade_base_001_002.sql
----------------------------------------------------------------------
diff --git a/libraries/kiwi/kiwi-triplestore/src/main/resources/org/apache/marmotta/kiwi/persistence/mysql/upgrade_base_001_002.sql
b/libraries/kiwi/kiwi-triplestore/src/main/resources/org/apache/marmotta/kiwi/persistence/mysql/upgrade_base_001_002.sql
index d72f1eb..52fdc09 100644
--- a/libraries/kiwi/kiwi-triplestore/src/main/resources/org/apache/marmotta/kiwi/persistence/mysql/upgrade_base_001_002.sql
+++ b/libraries/kiwi/kiwi-triplestore/src/main/resources/org/apache/marmotta/kiwi/persistence/mysql/upgrade_base_001_002.sql
@@ -1 +1,3 @@
-UPDATE metadata SET mvalue = '2' WHERE mkey = 'version';
\ No newline at end of file
+UPDATE metadata SET mvalue = '2' WHERE mkey = 'version';
+
+ALTER TABLE nodes ADD CONSTRAINT nodes_unique UNIQUE (ntype,svalue);

http://git-wip-us.apache.org/repos/asf/incubator-marmotta/blob/524e9a0d/libraries/kiwi/kiwi-triplestore/src/main/resources/org/apache/marmotta/kiwi/persistence/pgsql/create_base_tables.sql
----------------------------------------------------------------------
diff --git a/libraries/kiwi/kiwi-triplestore/src/main/resources/org/apache/marmotta/kiwi/persistence/pgsql/create_base_tables.sql
b/libraries/kiwi/kiwi-triplestore/src/main/resources/org/apache/marmotta/kiwi/persistence/pgsql/create_base_tables.sql
index 6fb4ad5..7f5f8c4 100644
--- a/libraries/kiwi/kiwi-triplestore/src/main/resources/org/apache/marmotta/kiwi/persistence/pgsql/create_base_tables.sql
+++ b/libraries/kiwi/kiwi-triplestore/src/main/resources/org/apache/marmotta/kiwi/persistence/pgsql/create_base_tables.sql
@@ -29,7 +29,8 @@ CREATE TABLE nodes (
   ltype     bigint     REFERENCES nodes(id),
   lang      varchar(5),
   createdAt timestamp  NOT NULL DEFAULT now(),
-  PRIMARY KEY(id)
+  PRIMARY KEY(id),
+  UNIQUE(ntype,svalue)
 );
 
 CREATE TABLE triples (
@@ -77,7 +78,7 @@ CREATE INDEX idx_namespaces_prefix ON namespaces(prefix);
 
 
 -- a rule to ignore duplicate inserts into triple table
-CREATE RULE "triples_ignore_duplicates" AS
+CREATE OR REPLACE RULE "triples_ignore_duplicates" AS
 ON INSERT TO triples
   WHERE EXISTS(
       SELECT 1 FROM triples

http://git-wip-us.apache.org/repos/asf/incubator-marmotta/blob/524e9a0d/libraries/kiwi/kiwi-triplestore/src/test/java/org/apache/marmotta/kiwi/test/H2ConcurrencyTest.java
----------------------------------------------------------------------
diff --git a/libraries/kiwi/kiwi-triplestore/src/test/java/org/apache/marmotta/kiwi/test/H2ConcurrencyTest.java
b/libraries/kiwi/kiwi-triplestore/src/test/java/org/apache/marmotta/kiwi/test/H2ConcurrencyTest.java
index e01be58..6cff442 100644
--- a/libraries/kiwi/kiwi-triplestore/src/test/java/org/apache/marmotta/kiwi/test/H2ConcurrencyTest.java
+++ b/libraries/kiwi/kiwi-triplestore/src/test/java/org/apache/marmotta/kiwi/test/H2ConcurrencyTest.java
@@ -22,6 +22,7 @@ import org.openrdf.repository.Repository;
 import org.openrdf.repository.RepositoryConnection;
 import org.openrdf.repository.RepositoryException;
 import org.openrdf.repository.sail.SailRepository;
+import org.openrdf.sail.SailException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -30,6 +31,8 @@ import java.util.ArrayList;
 import java.util.List;
 import java.util.Random;
 
+import static org.junit.Assert.assertTrue;
+
 /**
  * This test starts many triplestore operations in parallel to check if concurrent operations
will break things,
  *
@@ -115,7 +118,8 @@ public class H2ConcurrencyTest {
     }
 
     @AfterClass
-    public static void dropDatabase() throws RepositoryException, SQLException {
+    public static void dropDatabase() throws RepositoryException, SQLException, SailException
{
+        assertTrue(store.checkConsistency());
         store.closeValueFactory(); // release all connections before dropping the database
         store.getPersistence().dropDatabase();
         repository.shutDown();

http://git-wip-us.apache.org/repos/asf/incubator-marmotta/blob/524e9a0d/libraries/kiwi/kiwi-triplestore/src/test/java/org/apache/marmotta/kiwi/test/MySQLConcurrencyTest.java
----------------------------------------------------------------------
diff --git a/libraries/kiwi/kiwi-triplestore/src/test/java/org/apache/marmotta/kiwi/test/MySQLConcurrencyTest.java
b/libraries/kiwi/kiwi-triplestore/src/test/java/org/apache/marmotta/kiwi/test/MySQLConcurrencyTest.java
index 1b0ebcc..6fbb740 100644
--- a/libraries/kiwi/kiwi-triplestore/src/test/java/org/apache/marmotta/kiwi/test/MySQLConcurrencyTest.java
+++ b/libraries/kiwi/kiwi-triplestore/src/test/java/org/apache/marmotta/kiwi/test/MySQLConcurrencyTest.java
@@ -20,6 +20,7 @@ import org.openrdf.repository.Repository;
 import org.openrdf.repository.RepositoryConnection;
 import org.openrdf.repository.RepositoryException;
 import org.openrdf.repository.sail.SailRepository;
+import org.openrdf.sail.SailException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -28,6 +29,8 @@ import com.google.code.tempusfugit.concurrency.RepeatingRule;
 import com.google.code.tempusfugit.concurrency.annotations.Concurrent;
 import com.google.code.tempusfugit.concurrency.annotations.Repeating;
 
+import static org.junit.Assert.assertTrue;
+
 /**
  * This test starts many triplestore operations in parallel to check if concurrent operations
will break things,
  *
@@ -112,7 +115,8 @@ public class MySQLConcurrencyTest {
     }
 
     @AfterClass
-    public static void dropDatabase() throws RepositoryException, SQLException {
+    public static void dropDatabase() throws RepositoryException, SQLException, SailException
{
+        assertTrue(store.checkConsistency());
     	if (store != null) {
 	        store.closeValueFactory(); // release all connections before dropping the database
 	        store.getPersistence().dropDatabase();

http://git-wip-us.apache.org/repos/asf/incubator-marmotta/blob/524e9a0d/libraries/kiwi/kiwi-triplestore/src/test/java/org/apache/marmotta/kiwi/test/PostgreSQLConcurrencyTest.java
----------------------------------------------------------------------
diff --git a/libraries/kiwi/kiwi-triplestore/src/test/java/org/apache/marmotta/kiwi/test/PostgreSQLConcurrencyTest.java
b/libraries/kiwi/kiwi-triplestore/src/test/java/org/apache/marmotta/kiwi/test/PostgreSQLConcurrencyTest.java
index ef16598..b3c97c2 100644
--- a/libraries/kiwi/kiwi-triplestore/src/test/java/org/apache/marmotta/kiwi/test/PostgreSQLConcurrencyTest.java
+++ b/libraries/kiwi/kiwi-triplestore/src/test/java/org/apache/marmotta/kiwi/test/PostgreSQLConcurrencyTest.java
@@ -3,6 +3,7 @@ package org.apache.marmotta.kiwi.test;
 import java.sql.SQLException;
 import java.util.Random;
 
+import junit.framework.Assert;
 import org.apache.commons.lang3.RandomStringUtils;
 import org.apache.marmotta.kiwi.persistence.KiWiDialect;
 import org.apache.marmotta.kiwi.persistence.pgsql.PostgreSQLDialect;
@@ -20,6 +21,7 @@ import org.openrdf.repository.Repository;
 import org.openrdf.repository.RepositoryConnection;
 import org.openrdf.repository.RepositoryException;
 import org.openrdf.repository.sail.SailRepository;
+import org.openrdf.sail.SailException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -28,6 +30,8 @@ import com.google.code.tempusfugit.concurrency.RepeatingRule;
 import com.google.code.tempusfugit.concurrency.annotations.Concurrent;
 import com.google.code.tempusfugit.concurrency.annotations.Repeating;
 
+import static org.junit.Assert.assertTrue;
+
 /**
  * This test starts many triplestore operations in parallel to check if concurrent operations
will break things,
  *
@@ -112,7 +116,9 @@ public class PostgreSQLConcurrencyTest {
     }
 
     @AfterClass
-    public static void dropDatabase() throws RepositoryException, SQLException {
+    public static void dropDatabase() throws RepositoryException, SQLException, SailException
{
+        assertTrue(store.checkConsistency());
+
     	if (store != null) {
 	        store.closeValueFactory(); // release all connections before dropping the database
 	        store.getPersistence().dropDatabase();


Mime
View raw message