Mailing-List: contact oak-commits-help@jackrabbit.apache.org; run by ezmlm
Precedence: bulk
Reply-To: oak-dev@jackrabbit.apache.org
Content-Type: text/plain; charset="utf-8"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
Subject: svn commit: r1584287 - in /jackrabbit/oak/trunk/oak-core/src:
 main/java/org/apache/jackrabbit/oak/plugins/document/
 main/java/org/apache/jackrabbit/oak/plugins/document/mongo/
 test/java/org/apache/jackrabbit/oak/plugins/document/
Date: Thu, 03 Apr 2014 08:53:11 -0000
To: oak-commits@jackrabbit.apache.org
From: chetanm@apache.org
Message-Id: <20140403085311.C09B52388A3B@eris.apache.org>

Author: chetanm
Date: Thu Apr  3 08:53:10 2014
New Revision: 1584287

URL: http://svn.apache.org/r1584287
Log:
OAK-1295 - Recovery for missing _lastRev updates

-- Implemented the check for wether recovery is required by any node as Mongo query
-- Added a scheduled job for running the recovery check every 1 min by default

Also changed Clock handling by adding a resetToDefault method which resets the clock back to simple instead of doing that via passing null

Added:
    jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/LastRevRecoveryAgentTest.java   (with props)
Modified:
    jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/ClusterNodeInfo.java
    jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java
    jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreService.java
    jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/LastRevRecoveryAgent.java
    jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/MissingLastRevSeeker.java
    jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/Revision.java
    jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoMissingLastRevSeeker.java
    jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/ClusterInfoTest.java
    jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/LastRevRecoveryTest.java
    jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/LastRevSingleNodeRecoveryTest.java

Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/ClusterNodeInfo.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/ClusterNodeInfo.java?rev=1584287&r1=1584286&r2=1584287&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/ClusterNodeInfo.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/ClusterNodeInfo.java Thu Apr  3 08:53:10 2014
@@ -16,6 +16,7 @@
  */
 package org.apache.jackrabbit.oak.plugins.document;
 
+import static com.google.common.base.Preconditions.checkNotNull;
 import static org.apache.jackrabbit.oak.plugins.document.Document.ID;
 
 import java.lang.management.ManagementFactory;
@@ -58,7 +59,7 @@ public class ClusterNodeInfo {
     /**
      * The end of the lease.
      */
-    protected static final String LEASE_END_KEY = "leaseEnd";
+    public static final String LEASE_END_KEY = "leaseEnd";
 
     /**
      * The state of the cluster. On proper shutdown the state should be cleared.
@@ -135,11 +136,14 @@ public class ClusterNodeInfo {
      */
     private static Clock clock = Clock.SIMPLE;
 
+
+    public static final int DEFAULT_LEASE_DURATION_MILLIS = 1000 * 60;
+
     /**
      * The number of milliseconds for a lease (1 minute by default, and
      * initially).
      */
-    private long leaseTime = 1000 * 60;
+    private long leaseTime = DEFAULT_LEASE_DURATION_MILLIS;
 
     /**
      * The assigned cluster id.
@@ -410,17 +414,21 @@ public class ClusterNodeInfo {
 
     /**
      * Specify a custom clock to be used for determining current time.
-     * If passed clock is null then clock would be set to the default clock
      *
      * <b>Only Used For Testing</b>
      */
     static void setClock(Clock c) {
-        if(c == null){
-            c = Clock.SIMPLE;
-        }
+        checkNotNull(c);
         clock = c;
     }
 
+    /**
+     * Resets the clock to the default
+     */
+    static void resetClockToDefault(){
+        clock = Clock.SIMPLE;
+    }
+
     private static long getProcessId() {
         try {
             String name = ManagementFactory.getRuntimeMXBean().getName();

Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java?rev=1584287&r1=1584286&r2=1584287&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java Thu Apr  3 08:53:10 2014
@@ -538,6 +538,7 @@ public final class DocumentNodeStore
         return asyncDelay;
     }
 
+    @CheckForNull
     public ClusterNodeInfo getClusterInfo() {
         return clusterNodeInfo;
     }

Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreService.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreService.java?rev=1584287&r1=1584286&r2=1584287&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreService.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreService.java Thu Apr  3 08:53:10 2014
@@ -59,6 +59,7 @@ import org.apache.jackrabbit.oak.spi.sta
 import org.apache.jackrabbit.oak.spi.whiteboard.Registration;
 import org.apache.jackrabbit.oak.spi.whiteboard.Whiteboard;
 import org.apache.jackrabbit.oak.spi.whiteboard.WhiteboardExecutor;
+import org.apache.jackrabbit.oak.spi.whiteboard.WhiteboardUtils;
 import org.osgi.framework.Constants;
 import org.osgi.framework.ServiceRegistration;
 import org.osgi.service.component.ComponentContext;
@@ -138,6 +139,9 @@ public class DocumentNodeStoreService {
      */
     private long versionGcMaxAgeInSecs = DEFAULT_VER_GC_MAX_AGE;
 
+    public static final String PROP_REV_RECOVERY_INTERVAL = "lastRevRecoveryJobIntervalInSecs";
+
+
     @Activate
     protected void activate(ComponentContext context, Map<String, ?> config) throws Exception {
         this.context = context;
@@ -201,6 +205,7 @@ public class DocumentNodeStoreService {
         log.info("Connected to database {}", mongoDB);
 
         registerJMXBeans(mk.getNodeStore());
+        registerLastRevRecoveryJob(mk.getNodeStore());
 
         NodeStore store;
         if (useMK) {
@@ -220,7 +225,6 @@ public class DocumentNodeStoreService {
         reg = context.getBundleContext().registerService(NodeStore.class.getName(), store, props);
     }
 
-
     /**
      * At runtime DocumentNodeStore only pickup modification of certain properties
      */
@@ -315,8 +319,6 @@ public class DocumentNodeStoreService {
             );
         }
 
-
-
         if (blobStore instanceof GarbageCollectableBlobStore) {
             BlobGarbageCollector gc = new BlobGarbageCollector() {
                 @Override
@@ -340,6 +342,19 @@ public class DocumentNodeStoreService {
         //TODO Register JMX bean for Off Heap Cache stats
     }
 
+    private void registerLastRevRecoveryJob(final DocumentNodeStore nodeStore) {
+        long leaseTime = PropertiesUtil.toLong(context.getProperties().get(PROP_REV_RECOVERY_INTERVAL),
+                ClusterNodeInfo.DEFAULT_LEASE_DURATION_MILLIS);
+        Runnable recoverJob = new Runnable() {
+            @Override
+            public void run() {
+                nodeStore.getLastRevRecoveryAgent().performRecoveryIfNeeded();
+            }
+        };
+        registrations.add(WhiteboardUtils.scheduleWithFixedDelay(whiteboard,
+                recoverJob, TimeUnit.MILLISECONDS.toSeconds(leaseTime)));
+    }
+
     private Object prop(String propName) {
         return prop(propName, PREFIX + propName);
     }

Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/LastRevRecoveryAgent.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/LastRevRecoveryAgent.java?rev=1584287&r1=1584286&r2=1584287&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/LastRevRecoveryAgent.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/LastRevRecoveryAgent.java Thu Apr  3 08:53:10 2014
@@ -72,6 +72,11 @@ public class LastRevRecoveryAgent {
     public int recover(int clusterId) {
         ClusterNodeInfoDocument nodeInfo = missingLastRevUtil.getClusterNodeInfo(clusterId);
 
+        //TODO Currently leaseTime remains same per cluster node. If this
+        //is made configurable then it should be read from DB entry
+        final long leaseTime = ClusterNodeInfo.DEFAULT_LEASE_DURATION_MILLIS;
+        final long asyncDelay = nodeStore.getAsyncDelay();
+
         if (nodeInfo != null) {
             long leaseEnd = nodeInfo.getLeaseEndTime();
 
@@ -83,10 +88,17 @@ public class LastRevRecoveryAgent {
                 Revision lastRev = root.getLastRev().get(clusterId);
 
                 // start time is the _lastRev timestamp of this cluster node
-                long startTime = lastRev.getTimestamp();
+                final long startTime;
+                //lastRev can be null if other cluster node did not got
+                //chance to perform lastRev rollup even once
+                if (lastRev != null) {
+                    startTime = lastRev.getTimestamp();
+                } else {
+                    startTime = leaseEnd - leaseTime - asyncDelay;
+                }
 
                 // Endtime is the leaseEnd + the asyncDelay
-                long endTime = leaseEnd + nodeStore.getAsyncDelay();
+                long endTime = leaseEnd + asyncDelay;
 
                 log.info("Recovering candidates modified in time range : [{},{}] for clusterId [{}]",
                         Utils.timestampToString(startTime),
@@ -260,19 +272,40 @@ public class LastRevRecoveryAgent {
         }
         return null;
     }
+
+    /**
+     * Determines if any of the cluster node failed to renew its lease and
+     * did not properly shutdown. If any such cluster node is found then are potential
+     * candidates for last rev recovery
+     *
+     * @return true if last rev recovery needs to be performed for any of the cluster nodes
+     */
+    public boolean isRecoveryNeeded(){
+        return missingLastRevUtil.isRecoveryNeeded(nodeStore.getClock().getTime());
+    }
+
+    public void performRecoveryIfNeeded(){
+        if(isRecoveryNeeded()){
+            List<Integer> clusterIds = getRecoveryCandidateNodes();
+            log.info("Starting last revision recovery for following clusterId {}", clusterIds);
+            for(int clusterId : clusterIds){
+                recover(clusterId);
+            }
+        }
+    }
     
     /**
      * Gets the _lastRev recovery candidate cluster nodes.
      *
      * @return the recovery candidate nodes
      */
-    public List<String> getRecoveryCandidateNodes() {
+    public List<Integer> getRecoveryCandidateNodes() {
         Iterable<ClusterNodeInfoDocument> clusters = missingLastRevUtil.getAllClusters();
-        List<String> candidateClusterNodes = Lists.newArrayList();
+        List<Integer> candidateClusterNodes = Lists.newArrayList();
         
         for (ClusterNodeInfoDocument nodeInfo : clusters) {
             if (isRecoveryNeeded(nodeInfo)) {
-                candidateClusterNodes.add(nodeInfo.getId());
+                candidateClusterNodes.add(Integer.valueOf(nodeInfo.getId()));
             }
         }
         

Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/MissingLastRevSeeker.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/MissingLastRevSeeker.java?rev=1584287&r1=1584286&r2=1584287&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/MissingLastRevSeeker.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/MissingLastRevSeeker.java Thu Apr  3 08:53:10 2014
@@ -20,7 +20,6 @@
 package org.apache.jackrabbit.oak.plugins.document;
 
 import java.util.List;
-import java.util.concurrent.TimeUnit;
 
 import com.google.common.base.Predicate;
 import com.google.common.collect.Iterables;
@@ -77,8 +76,8 @@ public class MissingLastRevSeeker {
             public boolean apply(NodeDocument input) {
                 Long modified = (Long) input.get(NodeDocument.MODIFIED_IN_SECS);
                 return (modified != null
-                        && (modified > TimeUnit.MILLISECONDS.toSeconds(startTime))
-                        && (modified < TimeUnit.MILLISECONDS.toSeconds(endTime)));
+                        && (modified >= Commit.getModifiedInSecs(startTime))
+                        && (modified <= Commit.getModifiedInSecs(endTime)));
             }
         });
     }
@@ -101,5 +100,18 @@ public class MissingLastRevSeeker {
     public NodeDocument getRoot() {
         return store.find(Collection.NODES, Utils.getIdFromPath(ROOT_PATH));
     }
+
+    public boolean isRecoveryNeeded(long currentTime) {
+        for(ClusterNodeInfoDocument nodeInfo : getAllClusters()){
+            // Check if _lastRev recovery needed for this cluster node
+            // state is Active && currentTime past the leaseEnd time && recoveryLock not held by someone
+            if (nodeInfo.isActive()
+                    && currentTime > nodeInfo.getLeaseEndTime()
+                    && !nodeInfo.isBeingRecovered()) {
+                return true;
+            }
+        }
+        return false;
+    }
 }
 

Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/Revision.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/Revision.java?rev=1584287&r1=1584286&r2=1584287&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/Revision.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/Revision.java Thu Apr  3 08:53:10 2014
@@ -25,6 +25,9 @@ import java.util.concurrent.ConcurrentMa
 
 import org.apache.jackrabbit.oak.plugins.document.util.Utils;
 import org.apache.jackrabbit.oak.stats.Clock;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
 /**
  * A revision.
  */
@@ -71,10 +74,14 @@ public class Revision {
      * @param c - the clock
      */
     static void setClock(Clock c) {
+        checkNotNull(c);
         clock = c;
-        if (c == null) {
-            lastTimestamp = System.currentTimeMillis();
-        }
+    }
+
+    static void resetClockToDefault(){
+        clock = Clock.SIMPLE;
+        lastTimestamp = clock.getTime();
+
     }
     public Revision(long timestamp, int counter, int clusterId) {
         this(timestamp, counter, clusterId, false);

Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoMissingLastRevSeeker.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoMissingLastRevSeeker.java?rev=1584287&r1=1584286&r2=1584287&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoMissingLastRevSeeker.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoMissingLastRevSeeker.java Thu Apr  3 08:53:10 2014
@@ -101,6 +101,16 @@ public class MongoMissingLastRevSeeker e
         return oldNode != null;
     }
 
+    @Override
+    public boolean isRecoveryNeeded(long currentTime) {
+        QueryBuilder query =
+                start(ClusterNodeInfo.STATE).is(ClusterNodeInfo.ClusterNodeState.ACTIVE.name())
+                .put(ClusterNodeInfo.LEASE_END_KEY).lessThan(currentTime)
+                .put(ClusterNodeInfo.REV_RECOVERY_LOCK).notEquals(RecoverLockState.ACQUIRED.name());
+
+        return getClusterNodeCollection().findOne(query.get()) != null;
+    }
+
     private DBCollection getNodeCollection() {
         return store.getDBCollection(Collection.NODES);
     }

Modified: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/ClusterInfoTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/ClusterInfoTest.java?rev=1584287&r1=1584286&r2=1584287&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/ClusterInfoTest.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/ClusterInfoTest.java Thu Apr  3 08:53:10 2014
@@ -92,6 +92,6 @@ public class ClusterInfoTest {
 
     @After
     public void tearDown(){
-        ClusterNodeInfo.setClock(null);
+        ClusterNodeInfo.resetClockToDefault();
     }
 }

Added: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/LastRevRecoveryAgentTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/LastRevRecoveryAgentTest.java?rev=1584287&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/LastRevRecoveryAgentTest.java (added)
+++ jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/LastRevRecoveryAgentTest.java Thu Apr  3 08:53:10 2014
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.jackrabbit.oak.plugins.document;
+
+import java.io.IOException;
+import java.util.List;
+
+import com.google.common.collect.Lists;
+import org.apache.jackrabbit.oak.plugins.document.util.Utils;
+import org.apache.jackrabbit.oak.spi.commit.CommitInfo;
+import org.apache.jackrabbit.oak.spi.commit.EmptyHook;
+import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
+import org.apache.jackrabbit.oak.stats.Clock;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+@RunWith(Parameterized.class)
+public class LastRevRecoveryAgentTest {
+    private final DocumentStoreFixture fixture;
+
+    private DocumentNodeStore ds1;
+    private DocumentNodeStore ds2;
+    private int c1Id;
+    private int c2Id;
+    private DocumentStore sharedStore;
+    private Clock clock;
+
+    public LastRevRecoveryAgentTest(DocumentStoreFixture fixture) {
+        this.fixture = fixture;
+    }
+
+    //----------------------------------------< Set Up >
+
+    @Parameterized.Parameters
+    public static java.util.Collection<Object[]> fixtures() throws IOException {
+        List<Object[]> fixtures = Lists.newArrayList();
+        fixtures.add(new Object[] {new DocumentStoreFixture.MemoryFixture()});
+
+        DocumentStoreFixture mongo = new DocumentStoreFixture.MongoFixture();
+        if(mongo.isAvailable()){
+            fixtures.add(new Object[] {mongo});
+        }
+        return fixtures;
+    }
+
+    @Before
+    public void setUp() throws InterruptedException {
+        clock = new Clock.Virtual();
+
+        //Quite a bit of logic relies on timestamp converted
+        // to 5 sec resolutions
+        clock.waitUntil(System.currentTimeMillis());
+
+        ClusterNodeInfo.setClock(clock);
+        Revision.setClock(clock);
+        sharedStore = fixture.createDocumentStore();
+        ds1 = new DocumentMK.Builder()
+                .setAsyncDelay(0)
+                .clock(clock)
+                .setDocumentStore(sharedStore)
+                .getNodeStore();
+        c1Id = ds1.getClusterId();
+
+        ds2 = new DocumentMK.Builder()
+                .setAsyncDelay(0)
+                .clock(clock)
+                .setDocumentStore(sharedStore)
+                .getNodeStore();
+        c2Id = ds2.getClusterId();
+    }
+
+    @After
+    public void tearDown(){
+        sharedStore.dispose();
+        ClusterNodeInfo.resetClockToDefault();
+        Revision.resetClockToDefault();
+    }
+
+    //~------------------------------------------< Test Case >
+
+    @Test
+    public void testIsRecoveryRequired() throws Exception{
+        //1. Create base structure /x/y
+        NodeBuilder b1 = ds1.getRoot().builder();
+        b1.child("x").child("y");
+        ds1.merge(b1, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+        ds1.runBackgroundOperations();
+
+        ds2.runBackgroundOperations();
+
+        //2. Add a new node /x/y/z in C2
+        NodeBuilder b2 = ds2.getRoot().builder();
+        b2.child("x").child("y").child("z").setProperty("foo", "bar");
+        ds2.merge(b2, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+
+        NodeDocument z1 = getDocument(ds1, "/x/y/z");
+        Revision zlastRev2 = z1.getLastRev().get(c2Id);
+
+        long leaseTime = ds1.getClusterInfo().getLeaseTime();
+        ds1.runBackgroundOperations();
+
+        clock.waitUntil(clock.getTime() + leaseTime + 10);
+
+        //Renew the lease for C1
+        ds1.getClusterInfo().renewLease(3*leaseTime);
+
+        assertTrue(ds1.getLastRevRecoveryAgent().isRecoveryNeeded());
+
+        List<Integer> cids = ds1.getLastRevRecoveryAgent().getRecoveryCandidateNodes();
+        assertEquals(1, cids.size());
+        assertEquals(c2Id, cids.get(0).intValue());
+
+        ds1.getLastRevRecoveryAgent().recover(cids.get(0));
+
+        assertEquals(zlastRev2, getDocument(ds1, "/x/y").getLastRev().get(c2Id));
+        assertEquals(zlastRev2, getDocument(ds1, "/x").getLastRev().get(c2Id));
+        assertEquals(zlastRev2, getDocument(ds1, "/").getLastRev().get(c2Id));
+    }
+
+    private NodeDocument getDocument(DocumentNodeStore nodeStore, String path) {
+        return nodeStore.getDocumentStore().find(Collection.NODES, Utils.getIdFromPath(path));
+    }
+}

Propchange: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/LastRevRecoveryAgentTest.java
------------------------------------------------------------------------------
    svn:eol-style = native

Modified: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/LastRevRecoveryTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/LastRevRecoveryTest.java?rev=1584287&r1=1584286&r2=1584287&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/LastRevRecoveryTest.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/LastRevRecoveryTest.java Thu Apr  3 08:53:10 2014
@@ -35,22 +35,24 @@ import static org.junit.Assert.assertNul
 public class LastRevRecoveryTest {
     private DocumentNodeStore ds1;
     private DocumentNodeStore ds2;
+    private int c1Id;
+    private int c2Id;
     private MemoryDocumentStore sharedStore;
 
     @Before
     public void setUp(){
         sharedStore = new MemoryDocumentStore();
         ds1 = new DocumentMK.Builder()
-                .setClusterId(1)
                 .setAsyncDelay(0)
                 .setDocumentStore(sharedStore)
                 .getNodeStore();
+        c1Id = ds1.getClusterId();
 
         ds2 = new DocumentMK.Builder()
-                .setClusterId(2)
                 .setAsyncDelay(0)
                 .setDocumentStore(sharedStore)
                 .getNodeStore();
+        c2Id = ds2.getClusterId();
     }
 
 
@@ -84,21 +86,21 @@ public class LastRevRecoveryTest {
         NodeDocument y1 = getDocument(ds1, "/x/y");
         NodeDocument x1 = getDocument(ds1, "/x");
 
-        Revision zlastRev2 = z1.getLastRev().get(2);
+        Revision zlastRev2 = z1.getLastRev().get(c2Id);
         assertNotNull(zlastRev2);
 
         //lastRev should not be updated for C #2
-        assertNull(y1.getLastRev().get(2));
+        assertNull(y1.getLastRev().get(c2Id));
 
         LastRevRecoveryAgent recovery = new LastRevRecoveryAgent(ds1);
 
         //Do not pass y1 but still y1 should be updated
-        recovery.recover(Iterators.forArray(x1,z1), 2);
+        recovery.recover(Iterators.forArray(x1,z1), c2Id);
 
         //Post recovery the lastRev should be updated for /x/y and /x
-        assertEquals(zlastRev2, getDocument(ds1, "/x/y").getLastRev().get(2));
-        assertEquals(zlastRev2, getDocument(ds1, "/x").getLastRev().get(2));
-        assertEquals(zlastRev2, getDocument(ds1, "/").getLastRev().get(2));
+        assertEquals(zlastRev2, getDocument(ds1, "/x/y").getLastRev().get(c2Id));
+        assertEquals(zlastRev2, getDocument(ds1, "/x").getLastRev().get(c2Id));
+        assertEquals(zlastRev2, getDocument(ds1, "/").getLastRev().get(c2Id));
     }
 
     private NodeDocument getDocument(DocumentNodeStore nodeStore, String path) {

Modified: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/LastRevSingleNodeRecoveryTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/LastRevSingleNodeRecoveryTest.java?rev=1584287&r1=1584286&r2=1584287&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/LastRevSingleNodeRecoveryTest.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/LastRevSingleNodeRecoveryTest.java Thu Apr  3 08:53:10 2014
@@ -23,7 +23,6 @@ import static org.junit.Assert.assertEqu
 
 import java.io.IOException;
 import java.util.Collection;
-import java.util.Iterator;
 import java.util.List;
 
 import com.google.common.collect.Lists;
@@ -54,7 +53,6 @@ public class LastRevSingleNodeRecoveryTe
     @Parameterized.Parameters
     public static Collection<Object[]> fixtures() throws IOException {
         List<Object[]> fixtures = Lists.newArrayList();
-
         DocumentStoreFixture mongo = new DocumentStoreFixture.MongoFixture();
         if (mongo.isAvailable()) {
             fixtures.add(new Object[] {mongo});
@@ -170,9 +168,10 @@ public class LastRevSingleNodeRecoveryTe
         clock.waitUntil(clock.getTime() + mk.getClusterInfo().getLeaseTime() + 1000);
 
         LastRevRecoveryAgent recoveryAgent = mk.getNodeStore().getLastRevRecoveryAgent();
-        Iterator<String> iter = recoveryAgent.getRecoveryCandidateNodes().iterator();
-        assertEquals(String.valueOf(1), iter.next());
-        assertEquals(false, iter.hasNext());
+        List<Integer> cids = recoveryAgent.getRecoveryCandidateNodes();
+
+        assertEquals(1, cids.size());
+        assertEquals(Integer.valueOf(1), cids.get(0));
     }
     
     private void setupScenario() throws InterruptedException {
@@ -220,8 +219,8 @@ public class LastRevSingleNodeRecoveryTe
 
     @After
     public void tearDown() throws Exception {
-        Revision.setClock(null);
-        ClusterNodeInfo.setClock(null);
+        Revision.resetClockToDefault();
+        ClusterNodeInfo.resetClockToDefault();
         mk.dispose();
         fixture.dispose();
     }