Return-Path: X-Original-To: apmail-jackrabbit-oak-commits-archive@minotaur.apache.org Delivered-To: apmail-jackrabbit-oak-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id E8AD0EE1C for ; Fri, 1 Mar 2013 13:20:12 +0000 (UTC) Received: (qmail 64972 invoked by uid 500); 1 Mar 2013 13:20:12 -0000 Delivered-To: apmail-jackrabbit-oak-commits-archive@jackrabbit.apache.org Received: (qmail 64949 invoked by uid 500); 1 Mar 2013 13:20:12 -0000 Mailing-List: contact oak-commits-help@jackrabbit.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: oak-dev@jackrabbit.apache.org Delivered-To: mailing list oak-commits@jackrabbit.apache.org Received: (qmail 64928 invoked by uid 99); 1 Mar 2013 13:20:12 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 01 Mar 2013 13:20:12 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 01 Mar 2013 13:20:08 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 6228B23888E7; Fri, 1 Mar 2013 13:19:47 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1451586 - in /jackrabbit/oak/trunk/oak-mongomk/src: main/java/org/apache/jackrabbit/mongomk/prototype/ test/java/org/apache/jackrabbit/mongomk/prototype/ Date: Fri, 01 Mar 2013 13:19:46 -0000 To: oak-commits@jackrabbit.apache.org From: thomasm@apache.org X-Mailer: svnmailer-1.0.8-patched Message-Id: <20130301131947.6228B23888E7@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: thomasm Date: Fri Mar 1 13:19:46 2013 New Revision: 1451586 URL: http://svn.apache.org/r1451586 Log: OAK-619 Lock-free MongoMK implementation (bugfixes) Modified: jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/Commit.java jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/DocumentStore.java jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/MemoryDocumentStore.java jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/MongoDocumentStore.java jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/MongoMK.java jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/UpdateOp.java jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/Utils.java jackrabbit/oak/trunk/oak-mongomk/src/test/java/org/apache/jackrabbit/mongomk/prototype/SimpleTest.java Modified: jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/Commit.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/Commit.java?rev=1451586&r1=1451585&r2=1451586&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/Commit.java (original) +++ jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/Commit.java Fri Mar 1 13:19:46 2013 @@ -27,12 +27,16 @@ import org.apache.jackrabbit.mk.json.Jso import org.apache.jackrabbit.mk.json.JsopWriter; import org.apache.jackrabbit.mongomk.prototype.DocumentStore.Collection; import org.apache.jackrabbit.oak.commons.PathUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * A higher level object representing a commit. */ public class Commit { - + + private static final Logger LOG = LoggerFactory.getLogger(Commit.class); + /** * The maximum size of a document. If it is larger, it is split. */ @@ -93,7 +97,9 @@ public class Commit { void addNode(Node n) { if (operations.containsKey(n.path)) { - throw new MicroKernelException("Node already added: " + n.path); + String msg = "Node already added: " + n.path; + LOG.error(msg); + throw new MicroKernelException(msg); } operations.put(n.path, n.asOperation(true)); addedNodes.add(n.path); @@ -145,7 +151,13 @@ public class Commit { } try { if (newNodes.size() > 0) { - store.create(Collection.NODES, newNodes); + if (!store.create(Collection.NODES, newNodes)) { + for (UpdateOp op : newNodes) { + op.unset(UpdateOp.ID); + op.addMapEntry(UpdateOp.DELETED + "." + revision.toString(), "false"); + createOrUpdateNode(store, op); + } + } } for (UpdateOp op : changedNodes) { createOrUpdateNode(store, op); @@ -158,7 +170,9 @@ public class Commit { operations.put(commitRoot, root); } } catch (MicroKernelException e) { - throw new MicroKernelException("Exception committing " + diff.toString(), e); + String msg = "Exception committing " + diff.toString(); + LOG.error(msg, e); + throw new MicroKernelException(msg, e); } } @@ -193,8 +207,8 @@ public class Commit { } private UpdateOp[] splitDocument(Map map) { - String path = (String) map.get(UpdateOp.PATH); String id = (String) map.get(UpdateOp.ID); + String path = id.substring(1); Long previous = (Long) map.get(UpdateOp.PREVIOUS); if (previous == null) { previous = 0L; @@ -206,9 +220,7 @@ public class Commit { main.set(UpdateOp.PREVIOUS, previous); for (Entry e : map.entrySet()) { String key = e.getKey(); - if (key.equals(UpdateOp.PATH)) { - // ok - } else if (key.equals(UpdateOp.ID)) { + if (key.equals(UpdateOp.ID)) { // ok } else if (key.equals(UpdateOp.PREVIOUS)) { // ok @@ -278,6 +290,7 @@ public class Commit { UpdateOp op = operations.get(path); boolean isNew = op != null && op.isNew; boolean isWritten = op != null; + boolean isDelete = op != null && op.isDelete; long writeCountInc = mk.getWriteCountIncrement(path); Long writeCount = writeCounts.get(path); if (writeCount == null) { @@ -293,7 +306,7 @@ public class Commit { } } mk.applyChanges(revision, path, - isNew, isWritten, + isNew, isDelete, isWritten, writeCount, writeCountInc, added, removed); } @@ -328,6 +341,7 @@ public class Commit { public void removeNode(String path) { removedNodes.add(path); UpdateOp op = getUpdateOperationForNode(path); + op.setDelete(true); op.addMapEntry(UpdateOp.DELETED + "." + revision.toString(), "true"); long increment = mk.getWriteCountIncrement(path); op.increment(UpdateOp.WRITE_COUNT, 1 + increment); Modified: jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/DocumentStore.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/DocumentStore.java?rev=1451586&r1=1451585&r2=1451586&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/DocumentStore.java (original) +++ jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/DocumentStore.java Fri Mar 1 13:19:46 2013 @@ -49,7 +49,14 @@ public interface DocumentStore { */ void remove(Collection collection, String key); - void create(Collection collection, List updateOps); + /** + * Try to create a list of documents. + * + * @param collection the collection + * @param updateOps the list of documents to add + * @return true if this worked (if none of the documents already existed) + */ + boolean create(Collection collection, List updateOps); /** * Create or update a document. For MongoDb, this is using "findAndModify" with Modified: jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/MemoryDocumentStore.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/MemoryDocumentStore.java?rev=1451586&r1=1451585&r2=1451586&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/MemoryDocumentStore.java (original) +++ jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/MemoryDocumentStore.java Fri Mar 1 13:19:46 2013 @@ -177,10 +177,17 @@ public class MemoryDocumentStore impleme } @Override - public void create(Collection collection, List updateOps) { + public boolean create(Collection collection, List updateOps) { + ConcurrentSkipListMap> map = getMap(collection); + for (UpdateOp op : updateOps) { + if (map.containsKey(op.key)) { + return false; + } + } for (UpdateOp op : updateOps) { createOrUpdate(collection, op); } + return true; } public String toString() { Modified: jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/MongoDocumentStore.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/MongoDocumentStore.java?rev=1451586&r1=1451585&r2=1451586&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/MongoDocumentStore.java (original) +++ jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/MongoDocumentStore.java Fri Mar 1 13:19:46 2013 @@ -25,6 +25,8 @@ import java.util.Map.Entry; import org.apache.jackrabbit.mk.api.MicroKernelException; import org.apache.jackrabbit.mongomk.prototype.MongoMK.Cache; import org.apache.jackrabbit.mongomk.prototype.UpdateOp.Operation; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import com.mongodb.BasicDBObject; import com.mongodb.DB; @@ -40,9 +42,10 @@ import com.mongodb.WriteResult; * A document store that uses MongoDB as the backend. */ public class MongoDocumentStore implements DocumentStore { - - private static final boolean LOG = false; - private static final boolean LOG_TIME = true; + + private static final Logger LOG = LoggerFactory.getLogger(MongoDocumentStore.class); + + private static final boolean LOG_TIME = false; private final DBCollection nodesCollection; @@ -216,7 +219,7 @@ public class MongoDocumentStore implemen } @Override - public void create(Collection collection, List updateOps) { + public boolean create(Collection collection, List updateOps) { log("create", updateOps); ArrayList> maps = new ArrayList>(); DBObject[] inserts = new DBObject[updateOps.size()]; @@ -259,7 +262,7 @@ public class MongoDocumentStore implemen try { WriteResult writeResult = dbCollection.insert(inserts, WriteConcern.SAFE); if (writeResult.getError() != null) { - throw new MicroKernelException("Batch create failed: " + writeResult.getError()); + return false; } synchronized (cache) { for (Map map : maps) { @@ -269,8 +272,9 @@ public class MongoDocumentStore implemen } } } + return true; } catch (MongoException e) { - throw new MicroKernelException("Batch create failed", e); + return false; } } finally { end(start); @@ -319,19 +323,19 @@ public class MongoDocumentStore implemen @Override public void dispose() { - if (LOG_TIME) { - System.out.println("MongoDB time: " + time); + if (LOG.isDebugEnabled()) { + LOG.debug("MongoDB time: " + time); } nodesCollection.getDB().getMongo().close(); } - private static void log(Object... args) { - if (LOG) { - String msg = Arrays.toString(args); - if (msg.length() > 10000) { - msg = msg.length() + ": " + msg; + private static void log(String message, Object... args) { + if (LOG.isDebugEnabled()) { + String argList = Arrays.toString(args); + if (argList.length() > 10000) { + argList = argList.length() + ": " + argList; } - System.out.println(msg); + LOG.debug(message + argList); } } Modified: jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/MongoMK.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/MongoMK.java?rev=1451586&r1=1451585&r2=1451586&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/MongoMK.java (original) +++ jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/MongoMK.java Fri Mar 1 13:19:46 2013 @@ -268,21 +268,28 @@ public class MongoMK implements MicroKer @SuppressWarnings("unchecked") Map valueMap = (Map) v; if (valueMap != null) { - Revision latestRev = null; - for (String r : valueMap.keySet()) { - Revision propRev = Revision.fromString(r); - if (includeRevision(propRev, rev)) { - if (latestRev == null || isRevisionNewer(propRev, latestRev)) { - latestRev = propRev; - n.setProperty(key, valueMap.get(r)); - } - } - } + String value = getLatestValue(valueMap, rev); + n.setProperty(key, value); } } n.setWriteCount(writeCount); return n; } + + private String getLatestValue(Map valueMap, Revision rev) { + String value = null; + Revision latestRev = null; + for (String r : valueMap.keySet()) { + Revision propRev = Revision.fromString(r); + if (includeRevision(propRev, rev)) { + if (latestRev == null || isRevisionNewer(propRev, latestRev)) { + latestRev = propRev; + value = valueMap.get(r); + } + } + } + return value; + } @Override public String getHeadRevision() throws MicroKernelException { @@ -510,22 +517,15 @@ public class MongoMK implements MicroKer Map valueMap = (Map) nodeProps .get(UpdateOp.DELETED); if (valueMap != null) { - for (Map.Entry e : valueMap.entrySet()) { - // TODO What if multiple revisions are there?. Should we sort - // them and then - // determine include revision based on that - Revision propRev = Revision.fromString(e.getKey()); - if (includeRevision(propRev, rev)) { - if ("true".equals(e.getValue())) { - return true; - } - } + String value = getLatestValue(valueMap, rev); + if ("true".equals(value)) { + return true; } } return false; } - private static String stripBranchRevMarker(String revisionId){ + private static String stripBranchRevMarker(String revisionId) { if (revisionId.startsWith("b")) { return revisionId.substring(1); } @@ -679,7 +679,7 @@ public class MongoMK implements MicroKer } public void applyChanges(Revision rev, String path, - boolean isNew, boolean isWritten, + boolean isNew, boolean isDelete, boolean isWritten, long oldWriteCount, long writeCountInc, ArrayList added, ArrayList removed) { if (!isWritten) { @@ -693,7 +693,7 @@ public class MongoMK implements MicroKer } long newWriteCount = oldWriteCount + writeCountInc; Children c = nodeChildrenCache.get(path + "@" + (newWriteCount - 1)); - if (isNew || c != null) { + if (isNew || (!isDelete && c != null)) { String id = path + "@" + newWriteCount; Children c2 = new Children(path, id, rev); TreeSet set = new TreeSet(); Modified: jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/UpdateOp.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/UpdateOp.java?rev=1451586&r1=1451585&r2=1451586&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/UpdateOp.java (original) +++ jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/UpdateOp.java Fri Mar 1 13:19:46 2013 @@ -25,7 +25,6 @@ import java.util.TreeMap; public class UpdateOp { static final String ID = "_id"; - static final String PATH = "_path"; static final String WRITE_COUNT = "_writeCount"; static final String REVISIONS = "_revisions"; static final String PREVIOUS = "_prev"; @@ -36,6 +35,7 @@ public class UpdateOp { final String key; final boolean isNew; + boolean isDelete; final Map changes = new TreeMap(); @@ -46,6 +46,7 @@ public class UpdateOp { * @param path the node path (for nodes) * @param key the primary key * @param isNew whether this is a new document + * @param isDelete whether the _deleted property is set * @param rev the revision */ UpdateOp(String path, String key, boolean isNew) { @@ -62,6 +63,10 @@ public class UpdateOp { return isNew; } + void setDelete(boolean isDelete) { + this.isDelete = isDelete; + } + /** * Add a new map entry for this revision. * @@ -100,6 +105,15 @@ public class UpdateOp { op.value = value; changes.put(property, op); } + + /** + * Do not set the property (after it has been set). + * + * @param property the property name + */ + void unset(String property) { + changes.remove(property); + } /** * Increment the value. Modified: jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/Utils.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/Utils.java?rev=1451586&r1=1451585&r2=1451586&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/Utils.java (original) +++ jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/Utils.java Fri Mar 1 13:19:46 2013 @@ -20,6 +20,8 @@ import java.util.Map; import java.util.Map.Entry; import java.util.TreeMap; +import org.bson.types.ObjectId; + /** * Utility methods. */ @@ -49,5 +51,15 @@ public class Utils { } return size; } + + /** + * Generate a unique cluster id, similar to the machine id field in MongoDB ObjectId objects. + * + * @return the unique machine id + */ + public static int getUniqueClusterId() { + ObjectId objId = new ObjectId(); + return objId._machine(); + } } Modified: jackrabbit/oak/trunk/oak-mongomk/src/test/java/org/apache/jackrabbit/mongomk/prototype/SimpleTest.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-mongomk/src/test/java/org/apache/jackrabbit/mongomk/prototype/SimpleTest.java?rev=1451586&r1=1451585&r2=1451586&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-mongomk/src/test/java/org/apache/jackrabbit/mongomk/prototype/SimpleTest.java (original) +++ jackrabbit/oak/trunk/oak-mongomk/src/test/java/org/apache/jackrabbit/mongomk/prototype/SimpleTest.java Fri Mar 1 13:19:46 2013 @@ -58,17 +58,28 @@ public class SimpleTest { public void addNodeGetNode() { MongoMK mk = new MongoMK(); Revision rev = mk.newRevision(); - Node n = new Node("/", rev); + Node n = new Node("/test", rev); n.setProperty("name", "Hello"); UpdateOp op = n.asOperation(true); DocumentStore s = mk.getDocumentStore(); - s.create(Collection.NODES, Lists.newArrayList(op)); - Node n2 = mk.getNode("/", rev); + assertTrue(s.create(Collection.NODES, Lists.newArrayList(op))); + Node n2 = mk.getNode("/test", rev); assertEquals("Hello", n2.getProperty("name")); mk.dispose(); } @Test + public void reAddDeleted() { + MongoMK mk = createMK(); + String rev = mk.commit("/", "+\"test\":{\"name\": \"Hello\"}", null, null); + rev = mk.commit("/", "-\"test\"", rev, null); + rev = mk.commit("/", "+\"test\":{\"name\": \"Hallo\"}", null, null); + String test = mk.getNodes("/test", rev, 0, 0, Integer.MAX_VALUE, null); + assertEquals("{\"name\":\"Hallo\",\":childNodeCount\":0}", test); + mk.dispose(); + } + + @Test public void commit() { MongoMK mk = createMK();