jackrabbit-oak-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From chet...@apache.org
Subject svn commit: r1584333 - in /jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins: blob/MarkSweepGarbageCollector.java document/DocumentNodeStore.java document/DocumentNodeStoreService.java document/mongo/MongoBlobStore.java
Date Thu, 03 Apr 2014 10:02:35 GMT
Author: chetanm
Date: Thu Apr  3 10:02:34 2014
New Revision: 1584333

URL: http://svn.apache.org/r1584333
Log:
OAK-1639 - MarkSweepGarbageCollector improvements

-- Fixed log messages
-- Made the blob gc age configurable for DocumentNodeStore
-- Fixed an issue with GC logic which was using maxGCInterval as time

Modified:
    jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
    jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java
    jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreService.java
    jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoBlobStore.java

Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java?rev=1584333&r1=1584332&r2=1584333&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
(original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
Thu Apr  3 10:02:34 2014
@@ -20,6 +20,7 @@ import java.io.BufferedWriter;
 import java.io.File;
 import java.io.FileWriter;
 import java.io.IOException;
+import java.sql.Timestamp;
 import java.util.ArrayDeque;
 import java.util.Iterator;
 import java.util.List;
@@ -31,6 +32,7 @@ import java.util.concurrent.ConcurrentLi
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Executor;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
 
 import com.google.common.base.Charsets;
 import com.google.common.base.Joiner;
@@ -124,10 +126,6 @@ public class MarkSweepGarbageCollector i
 
     /**
      * Instantiates a new blob garbage collector.
-     * 
-     * @param marker
-     * @param blobStore
-     * @throws IOException Signals that an I/O exception has occurred.
      */
     public MarkSweepGarbageCollector(
             BlobReferenceRetriever marker, 
@@ -137,6 +135,15 @@ public class MarkSweepGarbageCollector i
         this(marker, blobStore, executor, TEMP_DIR, DEFAULT_BATCH_COUNT, true, TimeUnit.HOURS.toMillis(24));
     }
 
+    public MarkSweepGarbageCollector(
+            BlobReferenceRetriever marker,
+            GarbageCollectableBlobStore blobStore,
+            Executor executor,
+            long maxLastModifiedInterval)
+            throws IOException {
+        this(marker, blobStore, executor, TEMP_DIR, DEFAULT_BATCH_COUNT, true, maxLastModifiedInterval);
+    }
+
     @Override
     public void collectGarbage() throws Exception {
         markAndSweep();
@@ -297,6 +304,12 @@ public class MarkSweepGarbageCollector i
         return batchCount;
     }
 
+    private long getLastMaxModifiedTime(){
+        return maxLastModifiedInterval > 0 ?
+                System.currentTimeMillis() - maxLastModifiedInterval : 0;
+
+    }
+
     /**
      * Save batch to file.
      */
@@ -327,10 +340,7 @@ public class MarkSweepGarbageCollector i
         public void run() {
             try {
                 LOG.debug("Blob ids to be deleted {}", ids);
-                boolean deleted =
-                        blobStore.deleteChunks(ids,
-                                        (maxLastModifiedInterval > 0 ? System.currentTimeMillis()
-                                                - maxLastModifiedInterval : 0));
+                boolean deleted = blobStore.deleteChunks(ids,getLastMaxModifiedTime());
                 if (!deleted) {
                     exceptionQueue.addAll(ids);
                 }
@@ -346,13 +356,12 @@ public class MarkSweepGarbageCollector i
      */
     private void iterateNodeTree() throws IOException {
         final BufferedWriter writer = Files.newWriter(fs.getMarkedRefs(), Charsets.UTF_8);
+        final AtomicInteger count = new AtomicInteger();
         try {
             marker.collectReferences(
                     new ReferenceCollector() {
                         private final List<String> idBatch = Lists.newArrayListWithCapacity(getBatchCount());
 
-                        private int count = 0;
-
                         private final boolean debugMode = LOG.isTraceEnabled();
 
                         @Override
@@ -375,7 +384,7 @@ public class MarkSweepGarbageCollector i
                                     if (debugMode) {
                                         LOG.trace("chunkId : {}",id);
                                     }
-                                    count++;
+                                    count.getAndIncrement();
                                 }
 
                                 if (!idBatch.isEmpty()) {
@@ -385,13 +394,11 @@ public class MarkSweepGarbageCollector i
                             } catch (Exception e) {
                                 throw new RuntimeException("Error in retrieving references",
e);
                             }
-
-                            LOG.info("Number of valid blob references marked under mark phase
of " +
-                                    "Blob garbage collection [{}]",count);
                         }
                     }
             );
-
+            LOG.info("Number of valid blob references marked under mark phase of " +
+                    "Blob garbage collection [{}]",count.get());
             // sort the marked references
             fs.sort(fs.getMarkedRefs());
         } finally {
@@ -412,7 +419,7 @@ public class MarkSweepGarbageCollector i
             try {
                 bufferWriter = new BufferedWriter(
                         new FileWriter(fs.getAvailableRefs()));
-                Iterator<String> idsIter = blobStore.getAllChunkIds(maxLastModifiedInterval);
+                Iterator<String> idsIter = blobStore.getAllChunkIds(getLastMaxModifiedTime());
                 
                 List<String> ids = Lists.newArrayList();
 
@@ -431,7 +438,8 @@ public class MarkSweepGarbageCollector i
 
                 // sort the file
                 fs.sort(fs.getAvailableRefs());
-                LOG.debug("Ending retrieving all blobs : {}", blobsCount);
+                LOG.debug("Number of blobs present in BlobStore : [{}] which have " +
+                        "been last modified before [{}]", blobsCount, timestampToString(getLastMaxModifiedTime()));
             } finally {
                 IOUtils.closeQuietly(bufferWriter);
             }
@@ -575,4 +583,11 @@ public class MarkSweepGarbageCollector i
             throw new UnsupportedOperationException();
         }
     }
+
+    /**
+     * Provides a readable string for given timestamp
+     */
+    private static String timestampToString(long timestamp){
+        return (new Timestamp(timestamp) + "00").substring(0, 23);
+    }
 }

Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java?rev=1584333&r1=1584332&r2=1584333&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java
(original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java
Thu Apr  3 10:02:34 2014
@@ -39,6 +39,7 @@ import java.util.concurrent.Callable;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Executor;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.locks.ReadWriteLock;
@@ -1628,16 +1629,18 @@ public final class DocumentNodeStore
      * supports garbage collection
      *
      * @return garbage collector of the BlobStore supports GC otherwise null
+     * @param blobGcMaxAgeInSecs
      */
     @CheckForNull
-    public MarkSweepGarbageCollector createBlobGarbageCollector() {
+    public MarkSweepGarbageCollector createBlobGarbageCollector(long blobGcMaxAgeInSecs)
{
         MarkSweepGarbageCollector blobGC = null;
         if(blobStore instanceof GarbageCollectableBlobStore){
             try {
                 blobGC = new MarkSweepGarbageCollector(
                         new DocumentBlobReferenceRetriever(this),
                             (GarbageCollectableBlobStore) blobStore,
-                        executor);
+                        executor,
+                        TimeUnit.SECONDS.toMillis(blobGcMaxAgeInSecs));
             } catch (IOException e) {
                 throw new RuntimeException("Error occurred while initializing " +
                         "the MarkSweepGarbageCollector",e);

Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreService.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreService.java?rev=1584333&r1=1584332&r2=1584333&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreService.java
(original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreService.java
Thu Apr  3 10:02:34 2014
@@ -132,15 +132,22 @@ public class DocumentNodeStoreService {
     private Whiteboard whiteboard;
 
 
-    private static final long DEFAULT_VER_GC_MAX_AGE = TimeUnit.DAYS.toSeconds(1);
-    public static final String PROP_VER_GC_MAX_AGE = "versionGcMaxAgeInSecs";
     /**
      * Revisions older than this time would be garbage collected
      */
+    private static final long DEFAULT_VER_GC_MAX_AGE = TimeUnit.DAYS.toSeconds(1);
+    public static final String PROP_VER_GC_MAX_AGE = "versionGcMaxAgeInSecs";
     private long versionGcMaxAgeInSecs = DEFAULT_VER_GC_MAX_AGE;
 
     public static final String PROP_REV_RECOVERY_INTERVAL = "lastRevRecoveryJobIntervalInSecs";
 
+    /**
+     * Blob modified before this time duration would be considered for Blob GC
+     */
+    private static final long DEFAULT_BLOB_GC_MAX_AGE = TimeUnit.HOURS.toMillis(24);
+    public static final String PROP_BLOB_GC_MAX_AGE = "blobGcMaxAgeInSecs";
+    private long blobGcMaxAgeInSecs = DEFAULT_BLOB_GC_MAX_AGE;
+
 
     @Activate
     protected void activate(ComponentContext context, Map<String, ?> config) throws
Exception {
@@ -231,6 +238,7 @@ public class DocumentNodeStoreService {
     @Modified
     protected void modified(Map<String, ?> config){
         versionGcMaxAgeInSecs = PropertiesUtil.toLong(config.get(PROP_VER_GC_MAX_AGE), DEFAULT_VER_GC_MAX_AGE);
+        blobGcMaxAgeInSecs = PropertiesUtil.toLong(config.get(PROP_BLOB_GC_MAX_AGE), DEFAULT_BLOB_GC_MAX_AGE);
     }
 
     @Deactivate
@@ -323,7 +331,7 @@ public class DocumentNodeStoreService {
             BlobGarbageCollector gc = new BlobGarbageCollector() {
                 @Override
                 public void collectGarbage() throws Exception {
-                    store.createBlobGarbageCollector().collectGarbage();
+                    store.createBlobGarbageCollector(blobGcMaxAgeInSecs).collectGarbage();
                 }
             };
             registrations.add(registerMBean(whiteboard, BlobGCMBean.class, new BlobGC(gc,
executor),

Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoBlobStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoBlobStore.java?rev=1584333&r1=1584332&r2=1584333&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoBlobStore.java
(original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoBlobStore.java
Thu Apr  3 10:02:34 2014
@@ -231,6 +231,7 @@ public class MongoBlobStore extends Cach
                 collection.find(builder.get(), fields).hint(fields)
                         .addOption(Bytes.QUERYOPTION_SLAVEOK);
 
+        //TODO The cursor needs to be closed
         return new AbstractIterator<String>() {
             @Override
             protected String computeNext() {



Mime
View raw message