jackrabbit-oak-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From thom...@apache.org
Subject svn commit: r1451586 - in /jackrabbit/oak/trunk/oak-mongomk/src: main/java/org/apache/jackrabbit/mongomk/prototype/ test/java/org/apache/jackrabbit/mongomk/prototype/
Date Fri, 01 Mar 2013 13:19:46 GMT
Author: thomasm
Date: Fri Mar  1 13:19:46 2013
New Revision: 1451586

URL: http://svn.apache.org/r1451586
Log:
OAK-619 Lock-free MongoMK implementation (bugfixes)

Modified:
    jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/Commit.java
    jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/DocumentStore.java
    jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/MemoryDocumentStore.java
    jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/MongoDocumentStore.java
    jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/MongoMK.java
    jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/UpdateOp.java
    jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/Utils.java
    jackrabbit/oak/trunk/oak-mongomk/src/test/java/org/apache/jackrabbit/mongomk/prototype/SimpleTest.java

Modified: jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/Commit.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/Commit.java?rev=1451586&r1=1451585&r2=1451586&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/Commit.java
(original)
+++ jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/Commit.java
Fri Mar  1 13:19:46 2013
@@ -27,12 +27,16 @@ import org.apache.jackrabbit.mk.json.Jso
 import org.apache.jackrabbit.mk.json.JsopWriter;
 import org.apache.jackrabbit.mongomk.prototype.DocumentStore.Collection;
 import org.apache.jackrabbit.oak.commons.PathUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * A higher level object representing a commit.
  */
 public class Commit {
-    
+
+    private static final Logger LOG = LoggerFactory.getLogger(Commit.class);
+
     /**
      * The maximum size of a document. If it is larger, it is split.
      */
@@ -93,7 +97,9 @@ public class Commit {
 
     void addNode(Node n) {
         if (operations.containsKey(n.path)) {
-            throw new MicroKernelException("Node already added: " + n.path);
+            String msg = "Node already added: " + n.path;
+            LOG.error(msg);
+            throw new MicroKernelException(msg);
         }
         operations.put(n.path, n.asOperation(true));
         addedNodes.add(n.path);
@@ -145,7 +151,13 @@ public class Commit {
         }
         try {
             if (newNodes.size() > 0) {
-                store.create(Collection.NODES, newNodes);
+                if (!store.create(Collection.NODES, newNodes)) {
+                    for (UpdateOp op : newNodes) {
+                        op.unset(UpdateOp.ID);
+                        op.addMapEntry(UpdateOp.DELETED + "." + revision.toString(), "false");
+                        createOrUpdateNode(store, op);
+                    }
+                }
             }
             for (UpdateOp op : changedNodes) {
                 createOrUpdateNode(store, op);
@@ -158,7 +170,9 @@ public class Commit {
                 operations.put(commitRoot, root);
             }
         } catch (MicroKernelException e) {
-            throw new MicroKernelException("Exception committing " + diff.toString(), e);
+            String msg = "Exception committing " + diff.toString();
+            LOG.error(msg, e);
+            throw new MicroKernelException(msg, e);
         }
     }
     
@@ -193,8 +207,8 @@ public class Commit {
     }
     
     private UpdateOp[] splitDocument(Map<String, Object> map) {
-        String path = (String) map.get(UpdateOp.PATH);
         String id = (String) map.get(UpdateOp.ID);
+        String path = id.substring(1);
         Long previous = (Long) map.get(UpdateOp.PREVIOUS);
         if (previous == null) {
             previous = 0L;
@@ -206,9 +220,7 @@ public class Commit {
         main.set(UpdateOp.PREVIOUS, previous);
         for (Entry<String, Object> e : map.entrySet()) {
             String key = e.getKey();
-            if (key.equals(UpdateOp.PATH)) {
-                // ok
-            } else if (key.equals(UpdateOp.ID)) {
+            if (key.equals(UpdateOp.ID)) {
                 // ok
             } else if (key.equals(UpdateOp.PREVIOUS)) {
                 // ok
@@ -278,6 +290,7 @@ public class Commit {
             UpdateOp op = operations.get(path);
             boolean isNew = op != null && op.isNew;
             boolean isWritten = op != null;
+            boolean isDelete = op != null && op.isDelete;
             long writeCountInc = mk.getWriteCountIncrement(path);
             Long writeCount = writeCounts.get(path);
             if (writeCount == null) {
@@ -293,7 +306,7 @@ public class Commit {
                 }
             }
             mk.applyChanges(revision, path, 
-                    isNew, isWritten, 
+                    isNew, isDelete, isWritten, 
                     writeCount, writeCountInc,
                     added, removed);
         }
@@ -328,6 +341,7 @@ public class Commit {
     public void removeNode(String path) {
         removedNodes.add(path);
         UpdateOp op = getUpdateOperationForNode(path);
+        op.setDelete(true);
         op.addMapEntry(UpdateOp.DELETED + "." + revision.toString(), "true");
         long increment = mk.getWriteCountIncrement(path);
         op.increment(UpdateOp.WRITE_COUNT, 1 + increment);

Modified: jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/DocumentStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/DocumentStore.java?rev=1451586&r1=1451585&r2=1451586&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/DocumentStore.java
(original)
+++ jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/DocumentStore.java
Fri Mar  1 13:19:46 2013
@@ -49,7 +49,14 @@ public interface DocumentStore {
      */
     void remove(Collection collection, String key);
 
-    void create(Collection collection, List<UpdateOp> updateOps);
+    /**
+     * Try to create a list of documents.
+     * 
+     * @param collection the collection
+     * @param updateOps the list of documents to add
+     * @return true if this worked (if none of the documents already existed)
+     */
+    boolean create(Collection collection, List<UpdateOp> updateOps);
     
     /**
      * Create or update a document. For MongoDb, this is using "findAndModify" with

Modified: jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/MemoryDocumentStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/MemoryDocumentStore.java?rev=1451586&r1=1451585&r2=1451586&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/MemoryDocumentStore.java
(original)
+++ jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/MemoryDocumentStore.java
Fri Mar  1 13:19:46 2013
@@ -177,10 +177,17 @@ public class MemoryDocumentStore impleme
     }
 
     @Override
-    public void create(Collection collection, List<UpdateOp> updateOps) {
+    public boolean create(Collection collection, List<UpdateOp> updateOps) {
+        ConcurrentSkipListMap<String, Map<String, Object>> map = getMap(collection);
+        for (UpdateOp op : updateOps) {
+            if (map.containsKey(op.key)) {
+                return false;
+            }
+        }
         for (UpdateOp op : updateOps) {
             createOrUpdate(collection, op);
         }
+        return true;
     }
 
     public String toString() {

Modified: jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/MongoDocumentStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/MongoDocumentStore.java?rev=1451586&r1=1451585&r2=1451586&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/MongoDocumentStore.java
(original)
+++ jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/MongoDocumentStore.java
Fri Mar  1 13:19:46 2013
@@ -25,6 +25,8 @@ import java.util.Map.Entry;
 import org.apache.jackrabbit.mk.api.MicroKernelException;
 import org.apache.jackrabbit.mongomk.prototype.MongoMK.Cache;
 import org.apache.jackrabbit.mongomk.prototype.UpdateOp.Operation;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import com.mongodb.BasicDBObject;
 import com.mongodb.DB;
@@ -40,9 +42,10 @@ import com.mongodb.WriteResult;
  * A document store that uses MongoDB as the backend.
  */
 public class MongoDocumentStore implements DocumentStore {
-    
-    private static final boolean LOG = false;
-    private static final boolean LOG_TIME = true;
+
+    private static final Logger LOG = LoggerFactory.getLogger(MongoDocumentStore.class);
+
+    private static final boolean LOG_TIME = false;
 
     private final DBCollection nodesCollection;
     
@@ -216,7 +219,7 @@ public class MongoDocumentStore implemen
     }
 
     @Override
-    public void create(Collection collection, List<UpdateOp> updateOps) {
+    public boolean create(Collection collection, List<UpdateOp> updateOps) {
         log("create", updateOps);       
         ArrayList<Map<String, Object>> maps = new ArrayList<Map<String,
Object>>();
         DBObject[] inserts = new DBObject[updateOps.size()];
@@ -259,7 +262,7 @@ public class MongoDocumentStore implemen
             try {
                 WriteResult writeResult = dbCollection.insert(inserts, WriteConcern.SAFE);
                 if (writeResult.getError() != null) {
-                    throw new MicroKernelException("Batch create failed: " + writeResult.getError());
+                    return false;
                 }
                 synchronized (cache) {
                     for (Map<String, Object> map : maps) {
@@ -269,8 +272,9 @@ public class MongoDocumentStore implemen
                         }
                     }
                 }
+                return true;
             } catch (MongoException e) {
-                throw new MicroKernelException("Batch create failed", e);
+                return false;
             }
         } finally {
             end(start);
@@ -319,19 +323,19 @@ public class MongoDocumentStore implemen
     
     @Override
     public void dispose() {
-        if (LOG_TIME) {
-            System.out.println("MongoDB time: " + time);
+        if (LOG.isDebugEnabled()) {
+            LOG.debug("MongoDB time: " + time);
         }
         nodesCollection.getDB().getMongo().close();
     }
     
-    private static void log(Object... args) {
-        if (LOG) {
-            String msg = Arrays.toString(args);
-            if (msg.length() > 10000) {
-                msg = msg.length() + ": " + msg;
+    private static void log(String message, Object... args) {
+        if (LOG.isDebugEnabled()) {
+            String argList = Arrays.toString(args);
+            if (argList.length() > 10000) {
+                argList = argList.length() + ": " + argList;
             }
-            System.out.println(msg);
+            LOG.debug(message + argList);
         }
     }
 

Modified: jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/MongoMK.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/MongoMK.java?rev=1451586&r1=1451585&r2=1451586&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/MongoMK.java
(original)
+++ jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/MongoMK.java
Fri Mar  1 13:19:46 2013
@@ -268,21 +268,28 @@ public class MongoMK implements MicroKer
             @SuppressWarnings("unchecked")
             Map<String, String> valueMap = (Map<String, String>) v;
             if (valueMap != null) {
-                Revision latestRev = null;
-                for (String r : valueMap.keySet()) {
-                    Revision propRev = Revision.fromString(r);
-                    if (includeRevision(propRev, rev)) {
-                        if (latestRev == null || isRevisionNewer(propRev, latestRev)) {
-                            latestRev = propRev;
-                            n.setProperty(key, valueMap.get(r));
-                        }
-                    }
-                }
+                String value = getLatestValue(valueMap, rev);
+                n.setProperty(key, value);
             }
         }
         n.setWriteCount(writeCount);
         return n;
     }
+    
+    private String getLatestValue(Map<String, String> valueMap, Revision rev) {
+        String value = null;
+        Revision latestRev = null;
+        for (String r : valueMap.keySet()) {
+            Revision propRev = Revision.fromString(r);
+            if (includeRevision(propRev, rev)) {
+                if (latestRev == null || isRevisionNewer(propRev, latestRev)) {
+                    latestRev = propRev;
+                    value = valueMap.get(r);
+                }
+            }
+        }
+        return value;
+    }
 
     @Override
     public String getHeadRevision() throws MicroKernelException {
@@ -510,22 +517,15 @@ public class MongoMK implements MicroKer
         Map<String, String> valueMap = (Map<String, String>) nodeProps
                 .get(UpdateOp.DELETED);
         if (valueMap != null) {
-            for (Map.Entry<String, String> e : valueMap.entrySet()) {
-                // TODO What if multiple revisions are there?. Should we sort
-                // them and then
-                // determine include revision based on that
-                Revision propRev = Revision.fromString(e.getKey());
-                if (includeRevision(propRev, rev)) {
-                    if ("true".equals(e.getValue())) {
-                        return true;
-                    }
-                }
+            String value = getLatestValue(valueMap, rev);
+            if ("true".equals(value)) {
+                return true;
             }
         }
         return false;
     }
     
-    private static String stripBranchRevMarker(String revisionId){
+    private static String stripBranchRevMarker(String revisionId) {
         if (revisionId.startsWith("b")) {
             return revisionId.substring(1);
         }
@@ -679,7 +679,7 @@ public class MongoMK implements MicroKer
     }
 
     public void applyChanges(Revision rev, String path, 
-            boolean isNew, boolean isWritten, 
+            boolean isNew, boolean isDelete, boolean isWritten, 
             long oldWriteCount, long writeCountInc,
             ArrayList<String> added, ArrayList<String> removed) {
         if (!isWritten) {
@@ -693,7 +693,7 @@ public class MongoMK implements MicroKer
         }
         long newWriteCount = oldWriteCount + writeCountInc;
         Children c = nodeChildrenCache.get(path + "@" + (newWriteCount - 1));
-        if (isNew || c != null) {
+        if (isNew || (!isDelete && c != null)) {
             String id = path + "@" + newWriteCount;
             Children c2 = new Children(path, id, rev);
             TreeSet<String> set = new TreeSet<String>();

Modified: jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/UpdateOp.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/UpdateOp.java?rev=1451586&r1=1451585&r2=1451586&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/UpdateOp.java
(original)
+++ jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/UpdateOp.java
Fri Mar  1 13:19:46 2013
@@ -25,7 +25,6 @@ import java.util.TreeMap;
 public class UpdateOp {
     
     static final String ID = "_id";
-    static final String PATH = "_path";
     static final String WRITE_COUNT = "_writeCount";
     static final String REVISIONS = "_revisions";
     static final String PREVIOUS = "_prev";
@@ -36,6 +35,7 @@ public class UpdateOp {
     final String key;
     
     final boolean isNew;
+    boolean isDelete;
     
     final Map<String, Operation> changes = new TreeMap<String, Operation>();
     
@@ -46,6 +46,7 @@ public class UpdateOp {
      * @param path the node path (for nodes)
      * @param key the primary key
      * @param isNew whether this is a new document
+     * @param isDelete whether the _deleted property is set 
      * @param rev the revision
      */
     UpdateOp(String path, String key, boolean isNew) {
@@ -62,6 +63,10 @@ public class UpdateOp {
         return isNew;
     }
     
+    void setDelete(boolean isDelete) {
+        this.isDelete = isDelete;
+    }
+    
     /**
      * Add a new map entry for this revision.
      * 
@@ -100,6 +105,15 @@ public class UpdateOp {
         op.value = value;
         changes.put(property, op);
     }
+    
+    /**
+     * Do not set the property (after it has been set).
+     * 
+     * @param property the property name
+     */
+    void unset(String property) {
+        changes.remove(property);
+    }
 
     /**
      * Increment the value.

Modified: jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/Utils.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/Utils.java?rev=1451586&r1=1451585&r2=1451586&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/Utils.java
(original)
+++ jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/Utils.java
Fri Mar  1 13:19:46 2013
@@ -20,6 +20,8 @@ import java.util.Map;
 import java.util.Map.Entry;
 import java.util.TreeMap;
 
+import org.bson.types.ObjectId;
+
 /**
  * Utility methods.
  */
@@ -49,5 +51,15 @@ public class Utils {
         }
         return size;
     }
+
+    /**
+     * Generate a unique cluster id, similar to the machine id field in MongoDB ObjectId
objects.
+     * 
+     * @return the unique machine id
+     */
+    public static int getUniqueClusterId() {
+        ObjectId objId = new ObjectId();
+        return objId._machine();
+    }
     
 }

Modified: jackrabbit/oak/trunk/oak-mongomk/src/test/java/org/apache/jackrabbit/mongomk/prototype/SimpleTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-mongomk/src/test/java/org/apache/jackrabbit/mongomk/prototype/SimpleTest.java?rev=1451586&r1=1451585&r2=1451586&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-mongomk/src/test/java/org/apache/jackrabbit/mongomk/prototype/SimpleTest.java
(original)
+++ jackrabbit/oak/trunk/oak-mongomk/src/test/java/org/apache/jackrabbit/mongomk/prototype/SimpleTest.java
Fri Mar  1 13:19:46 2013
@@ -58,17 +58,28 @@ public class SimpleTest {
     public void addNodeGetNode() {
         MongoMK mk = new MongoMK();
         Revision rev = mk.newRevision();
-        Node n = new Node("/", rev);
+        Node n = new Node("/test", rev);
         n.setProperty("name", "Hello");
         UpdateOp op = n.asOperation(true);
         DocumentStore s = mk.getDocumentStore();
-        s.create(Collection.NODES, Lists.newArrayList(op));
-        Node n2 = mk.getNode("/", rev);
+        assertTrue(s.create(Collection.NODES, Lists.newArrayList(op)));
+        Node n2 = mk.getNode("/test", rev);
         assertEquals("Hello", n2.getProperty("name"));
         mk.dispose();
     }
 
     @Test
+    public void reAddDeleted() {
+        MongoMK mk = createMK();
+        String rev = mk.commit("/", "+\"test\":{\"name\": \"Hello\"}", null, null);
+        rev = mk.commit("/", "-\"test\"", rev, null);
+        rev = mk.commit("/", "+\"test\":{\"name\": \"Hallo\"}", null, null);
+        String test = mk.getNodes("/test", rev, 0, 0, Integer.MAX_VALUE, null);
+        assertEquals("{\"name\":\"Hallo\",\":childNodeCount\":0}", test);
+        mk.dispose();
+    }
+    
+    @Test
     public void commit() {
         MongoMK mk = createMK();
         



Mime
View raw message