jackrabbit-oak-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From resc...@apache.org
Subject svn commit: r1584385 - in /jackrabbit/oak/trunk/oak-core/src: main/java/org/apache/jackrabbit/oak/plugins/document/rdb/ test/java/org/apache/jackrabbit/oak/plugins/document/
Date Thu, 03 Apr 2014 20:34:55 GMT
Author: reschke
Date: Thu Apr  3 20:34:55 2014
New Revision: 1584385

URL: http://svn.apache.org/r1584385
Log:
OAK-1590 - add very basic performance and concurrency tests; add retries for crate/update
race conditions (WIP)

Modified:
    jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStore.java
    jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/AbstractDocumentStoreTest.java
    jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/BasicDocumentStoreTest.java
    jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentStoreFixture.java

Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStore.java?rev=1584385&r1=1584384&r2=1584385&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStore.java
(original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStore.java
Thu Apr  3 20:34:55 2014
@@ -155,8 +155,8 @@ public class RDBDocumentStore implements
 
     @Override
     public <T extends Document> void remove(Collection<T> collection, List<String>
keys) {
-        //TODO Use batch delete
-        for(String key : keys){
+        // TODO Use batch delete
+        for (String key : keys) {
             remove(collection, key);
         }
     }
@@ -234,6 +234,9 @@ public class RDBDocumentStore implements
     // string length at which we switch to BLOB storage
     private static int DATALIMIT = 16384 / 4;
 
+    // number of retries for updates
+    private static int RETRIES = 10;
+
     private void initialize(DataSource ds, DocumentMK.Builder builder) throws Exception {
 
         this.ds = ds;
@@ -247,13 +250,12 @@ public class RDBDocumentStore implements
         try {
             con.setAutoCommit(false);
 
-            for (String tableName : new String[] { "CLUSTERNODES", "NODES", "SETTINGS"})
{
+            for (String tableName : new String[] { "CLUSTERNODES", "NODES", "SETTINGS" })
{
                 try {
                     PreparedStatement stmt = con.prepareStatement("select ID from " + tableName
+ " where ID = ?");
                     stmt.setString(1, "0:/");
                     stmt.executeQuery();
-                }
-                catch(SQLException ex) {
+                } catch (SQLException ex) {
                     // table does not appear to exist
                     con.rollback();
 
@@ -262,17 +264,19 @@ public class RDBDocumentStore implements
 
                     Statement stmt = con.createStatement();
 
-                    // the code below likely will need to be extended for new database types
+                    // the code below likely will need to be extended for new
+                    // database types
                     if ("PostgreSQL".equals(dbtype)) {
-                        stmt.execute("create table " + tableName
+                        stmt.execute("create table "
+                                + tableName
                                 + " (ID varchar(1000) not null primary key, MODIFIED bigint,
MODCOUNT bigint, SIZE bigint, DATA varchar(16384), BDATA bytea)");
-                    }
-                    else if ("DB2".equals(dbtype) || (dbtype != null && dbtype.startsWith("DB2/")))
{
-                        stmt.execute("create table " + tableName
+                    } else if ("DB2".equals(dbtype) || (dbtype != null && dbtype.startsWith("DB2/")))
{
+                        stmt.execute("create table "
+                                + tableName
                                 + " (ID varchar(1000) not null primary key, MODIFIED bigint,
MODCOUNT bigint, SIZE bigint, DATA varchar(16384), BDATA blob)");
-                    }
-                    else {
-                        stmt.execute("create table " + tableName
+                    } else {
+                        stmt.execute("create table "
+                                + tableName
                                 + " (ID varchar(1000) not null primary key, MODIFIED bigint,
MODCOUNT bigint, SIZE bigint, DATA varchar(16384), BDATA blob)");
                     }
                     stmt.close();
@@ -325,15 +329,32 @@ public class RDBDocumentStore implements
             }
             update.increment(MODCOUNT, 1);
             UpdateUtils.applyChanges(doc, update, comparator);
-            insertDocument(collection, doc);
-            addToCache(collection, doc);
-        } else {
-            T doc = applyChanges(collection, oldDoc, update, checkConditions);
-            if (doc == null) {
-                return null;
+            try {
+                insertDocument(collection, doc);
+                addToCache(collection, doc);
+                return oldDoc;
             }
+            catch (MicroKernelException ex) {
+                // may have failed due to a race condition; try update instead
+                oldDoc = readDocument(collection, update.getId());
+                if (oldDoc == null) {
+                    // something else went wrong
+                    throw(ex);
+                }
+                return internalUpdate(collection, update, oldDoc, checkConditions, RETRIES);
+            }
+        } else {
+            return internalUpdate(collection, update, oldDoc, checkConditions, RETRIES);
+        }
+    }
 
-            int retries = 5; // TODO
+    @CheckForNull
+    private <T extends Document> T internalUpdate(Collection<T> collection, UpdateOp
update, T oldDoc, boolean checkConditions,
+            int retries) {
+        T doc = applyChanges(collection, oldDoc, update, checkConditions);
+        if (doc == null) {
+            return null;
+        } else {
             boolean success = false;
 
             while (!success && retries > 0) {
@@ -354,9 +375,9 @@ public class RDBDocumentStore implements
             if (!success) {
                 throw new MicroKernelException("failed update (race?)");
             }
-        }
 
-        return oldDoc;
+            return oldDoc;
+        }
     }
 
     @CheckForNull
@@ -541,6 +562,7 @@ public class RDBDocumentStore implements
             dbInsert(connection, tableName, document.getId(), modified, modcount, data);
             connection.commit();
         } catch (SQLException ex) {
+            LOG.debug("insert of " + document.getId() + " failed", ex);
             try {
                 connection.rollback();
             } catch (SQLException e) {

Modified: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/AbstractDocumentStoreTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/AbstractDocumentStoreTest.java?rev=1584385&r1=1584384&r2=1584385&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/AbstractDocumentStoreTest.java
(original)
+++ jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/AbstractDocumentStoreTest.java
Thu Apr  3 20:34:55 2014
@@ -18,7 +18,10 @@ package org.apache.jackrabbit.oak.plugin
 
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.HashSet;
+import java.util.Set;
 
+import org.junit.After;
 import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
@@ -27,12 +30,24 @@ public abstract class AbstractDocumentSt
 
     protected String dsname;
     protected DocumentStore ds;
+    protected Set<String> removeMe = new HashSet<String>();
 
     public AbstractDocumentStoreTest(DocumentStoreFixture dsf) {
         this.ds = dsf.createDocumentStore();
         this.dsname = dsf.getName();
     }
 
+    @After
+    public void cleanUp() {
+        for (String id : removeMe) {
+            try {
+                ds.remove(org.apache.jackrabbit.oak.plugins.document.Collection.NODES, id);
+            } catch (Exception ex) {
+                // best effort
+            }
+        }
+    }
+
     @Parameterized.Parameters
     public static Collection<Object[]> fixtures() {
         Collection<Object[]> result = new ArrayList<Object[]>();

Modified: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/BasicDocumentStoreTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/BasicDocumentStoreTest.java?rev=1584385&r1=1584384&r2=1584385&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/BasicDocumentStoreTest.java
(original)
+++ jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/BasicDocumentStoreTest.java
Thu Apr  3 20:34:55 2014
@@ -20,11 +20,8 @@ import static org.junit.Assert.assertNot
 import static org.junit.Assert.assertTrue;
 
 import java.util.Collections;
-import java.util.HashSet;
-import java.util.Set;
 
 import org.apache.jackrabbit.oak.plugins.document.mongo.MongoDocumentStore;
-import org.junit.After;
 import org.junit.Assume;
 import org.junit.Test;
 import org.slf4j.Logger;
@@ -34,23 +31,10 @@ public class BasicDocumentStoreTest exte
 
     private static final Logger LOG = LoggerFactory.getLogger(BasicDocumentStoreTest.class);
 
-    private Set<String> removeMe = new HashSet<String>();
-
     public BasicDocumentStoreTest(DocumentStoreFixture dsf) {
         super(dsf);
     }
 
-    @After
-    public void cleanUp() {
-        for (String id : removeMe) {
-            try {
-                super.ds.remove(Collection.NODES, id);
-            } catch (Exception ex) {
-                // best effort
-            }
-        }
-    }
-
     @Test
     public void testAddAndRemove() {
         String id = this.getClass().getName() + ".testAddAndRemove";
@@ -78,7 +62,7 @@ public class BasicDocumentStoreTest exte
 
         while (max - min >= 2) {
             test = (max + min) / 2;
-            String id = generateId(test);
+            String id = generateString(test);
             UpdateOp up = new UpdateOp(id, true);
             up.set("_id", id);
             boolean success = super.ds.create(Collection.NODES, Collections.singletonList(up));
@@ -105,7 +89,7 @@ public class BasicDocumentStoreTest exte
         while (max - min >= 256) {
             test = (max + min) / 2;
             String id = this.getClass().getName() + ".testMaxProperty-" + test;
-            String pval = generateId(test);
+            String pval = generateString(test);
             UpdateOp up = new UpdateOp(id, true);
             up.set("_id", id);
             up.set("foo", pval);
@@ -124,10 +108,72 @@ public class BasicDocumentStoreTest exte
         LOG.info("max prop length for " + super.dsname + " was " + test);
     }
 
-    private static String generateId(int length) {
+    @Test
+    public void testCreatePerfSmall() {
+        createPerf(16);
+    }
+
+    @Test
+    public void testCreatePerfBig() {
+        createPerf(32 * 1024);
+    }
+
+    private void createPerf(int size) {
+        String pval = generateString(size);
+        long duration = 1000;
+        long end = System.currentTimeMillis() + duration;
+        long cnt = 0;
+
+        while (System.currentTimeMillis() < end) {
+            String id = this.getClass().getName() + ".testCreatePerf-" + size + "-" + cnt;
+            UpdateOp up = new UpdateOp(id, true);
+            up.set("_id", id);
+            up.set("foo", pval);
+            boolean success = super.ds.create(Collection.NODES, Collections.singletonList(up));
+            assertTrue("document with " + id + " nit created", success);
+            removeMe.add(id);
+            cnt += 1;
+        }
+
+        LOG.info("document creation with property of size " + size + " for " + super.dsname
+ " was " + cnt + " in " + duration + "ms ("
+                + (cnt / (duration / 1000f)) + "/s)");
+    }
+
+    @Test
+    public void testUpdatePerfSmall() {
+        updatePerf(16);
+    }
+
+    @Test
+    public void testUpdatePerfBig() {
+        updatePerf(32 * 1024);
+    }
+
+    private void updatePerf(int size) {
+        String pval = generateString(size);
+        long duration = 1000;
+        long end = System.currentTimeMillis() + duration;
+        long cnt = 0;
+
+        String id = this.getClass().getName() + ".testUpdatePerf-" + size;
+        removeMe.add(id);
+
+        while (System.currentTimeMillis() < end) {
+            UpdateOp up = new UpdateOp(id, true);
+            up.set("_id", id);
+            up.set("foo", pval);
+            super.ds.createOrUpdate(Collection.NODES, up);
+            cnt += 1;
+        }
+
+        LOG.info("document updates with property of size " + size + " for " + super.dsname
+ " was " + cnt + " in " + duration + "ms ("
+                + (cnt / (duration / 1000f)) + "/s)");
+    }
+
+    private static String generateString(int length) {
         StringBuffer buf = new StringBuffer(length);
         while (length-- > 0) {
-            buf.append('A' + ((int)(26 * Math.random())));
+            buf.append('A' + ((int) (26 * Math.random())));
         }
         return buf.toString();
     }

Modified: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentStoreFixture.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentStoreFixture.java?rev=1584385&r1=1584384&r2=1584385&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentStoreFixture.java
(original)
+++ jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentStoreFixture.java
Thu Apr  3 20:34:55 2014
@@ -34,7 +34,7 @@ public abstract class DocumentStoreFixtu
     private static final Logger LOG = LoggerFactory.getLogger(DocumentStoreFixture.class);
 
     public static final DocumentStoreFixture MEMORY = new MemoryFixture();
-    public static final DocumentStoreFixture RDB_H2 = new RDBFixture("RDB-Postgres", "jdbc:h2:file:./target/ds-test",
"sa", "");
+    public static final DocumentStoreFixture RDB_H2 = new RDBFixture("RDB-H2(file)", "jdbc:h2:file:./target/ds-test",
"sa", "");
     public static final DocumentStoreFixture RDB_PG = new RDBFixture("RDB-Postgres", "jdbc:postgresql:oak",
"postgres", "geheim");
     public static final DocumentStoreFixture RDB_DB2 = new RDBFixture("RDB-DB2", "jdbc:db2://localhost:50000/OAK",
"oak", "geheim");
     public static final DocumentStoreFixture MONGO = new MongoFixture("mongodb://localhost:27017/oak");
@@ -134,7 +134,7 @@ public abstract class DocumentStoreFixtu
                 return false;
             }
         }
-        
+
         @Override
         public void dispose() {
             try{



Mime
View raw message