jackrabbit-oak-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From thom...@apache.org
Subject svn commit: r1571268 - in /jackrabbit/oak/trunk: oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/ oak-blob/src/test/java/org/apache/jackrabbit/oak/spi/blob/ oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/sort/ oak-core/src/main/j...
Date Mon, 24 Feb 2014 12:53:41 GMT
Author: thomasm
Date: Mon Feb 24 12:53:40 2014
New Revision: 1571268

URL: http://svn.apache.org/r1571268
Log:
OAK-377 Data store garbage collection

Modified:
    jackrabbit/oak/trunk/oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/FileBlobStore.java
    jackrabbit/oak/trunk/oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/GarbageCollectableBlobStore.java
    jackrabbit/oak/trunk/oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/MemoryBlobStore.java
    jackrabbit/oak/trunk/oak-blob/src/test/java/org/apache/jackrabbit/oak/spi/blob/AbstractBlobStoreTest.java
    jackrabbit/oak/trunk/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/sort/ExternalSort.java
    jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
    jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/cloud/CloudBlobStore.java
    jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java
    jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/db/DbBlobStore.java
    jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoBlobStore.java
    jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBBlobStore.java
    jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/MongoBlobGCTest.java
    jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/blob/AbstractBlobStoreTest.java

Modified: jackrabbit/oak/trunk/oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/FileBlobStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/FileBlobStore.java?rev=1571268&r1=1571267&r2=1571268&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/FileBlobStore.java
(original)
+++ jackrabbit/oak/trunk/oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/FileBlobStore.java
Mon Feb 24 12:53:40 2014
@@ -219,7 +219,7 @@ public class FileBlobStore extends Abstr
     }
 
     @Override
-    public boolean deleteChunk(String chunkId) throws Exception {
+    public boolean deleteChunk(String chunkId, long maxLastModifiedTime) throws Exception
{
         byte[] digest = StringUtils.convertHexToBytes(chunkId);
         File f = getFile(digest, false);
         if (!f.exists()) {
@@ -228,8 +228,11 @@ public class FileBlobStore extends Abstr
             old.renameTo(f);
             f = getFile(digest, false);
         }
-        f.delete();
-        return mark;
+        if ((maxLastModifiedTime <= 0) 
+                || FileUtils.isFileOlder(f, maxLastModifiedTime)) {
+            return f.delete();
+        }
+        return false;
     }
 
     @Override
@@ -241,8 +244,8 @@ public class FileBlobStore extends Abstr
                     @Override
                     public boolean apply(@Nullable File input) {
                         if (!input.isDirectory() && (
-                                (maxLastModifiedTime == 0 || maxLastModifiedTime == -1) ||
-                                FileUtils.isFileOlder(input, maxLastModifiedTime))) {
+                                (maxLastModifiedTime <= 0)
+                                    || FileUtils.isFileOlder(input, maxLastModifiedTime)))
{
                             return true;
                         }
                         return false;

Modified: jackrabbit/oak/trunk/oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/GarbageCollectableBlobStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/GarbageCollectableBlobStore.java?rev=1571268&r1=1571267&r2=1571268&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/GarbageCollectableBlobStore.java
(original)
+++ jackrabbit/oak/trunk/oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/GarbageCollectableBlobStore.java
Mon Feb 24 12:53:40 2014
@@ -86,11 +86,13 @@ public interface GarbageCollectableBlobS
      * Delete the blob with the given id.
      * 
      * @param chunkId the chunk id
+     * @param maxLastModifiedTime
+     *            the max last modified time to consider for retrieval 
      * @return true, if successful
      * @throws Exception
      *             the exception
      */
-    boolean deleteChunk(String chunkId) throws Exception;
+    boolean deleteChunk(String chunkId, long maxLastModifiedTime) throws Exception;
 
     /**
      * Resolve chunks from the given Id.

Modified: jackrabbit/oak/trunk/oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/MemoryBlobStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/MemoryBlobStore.java?rev=1571268&r1=1571267&r2=1571268&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/MemoryBlobStore.java
(original)
+++ jackrabbit/oak/trunk/oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/MemoryBlobStore.java
Mon Feb 24 12:53:40 2014
@@ -80,8 +80,11 @@ public class MemoryBlobStore extends Abs
         return count;
     }
 
+    /**
+     * Ignores the maxlastModifiedTime
+     */
     @Override
-    public boolean deleteChunk(String chunkId) throws Exception {
+    public boolean deleteChunk(String chunkId, long maxLastModifiedTime) throws Exception
{
         BlockId id = new BlockId(StringUtils.convertHexToBytes(chunkId), 0);
         if (map.containsKey(id)) {
             map.remove(id);

Modified: jackrabbit/oak/trunk/oak-blob/src/test/java/org/apache/jackrabbit/oak/spi/blob/AbstractBlobStoreTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob/src/test/java/org/apache/jackrabbit/oak/spi/blob/AbstractBlobStoreTest.java?rev=1571268&r1=1571267&r2=1571268&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-blob/src/test/java/org/apache/jackrabbit/oak/spi/blob/AbstractBlobStoreTest.java
(original)
+++ jackrabbit/oak/trunk/oak-blob/src/test/java/org/apache/jackrabbit/oak/spi/blob/AbstractBlobStoreTest.java
Mon Feb 24 12:53:40 2014
@@ -373,7 +373,7 @@ public abstract class AbstractBlobStoreT
         Set<String> ids = createArtifacts();
 
         for (String id : ids) {
-            store.deleteChunk(id);
+            store.deleteChunk(id, 0);
         }
 
         Iterator<String> iter = store.getAllChunkIds(0);

Modified: jackrabbit/oak/trunk/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/sort/ExternalSort.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/sort/ExternalSort.java?rev=1571268&r1=1571267&r2=1571268&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/sort/ExternalSort.java
(original)
+++ jackrabbit/oak/trunk/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/sort/ExternalSort.java
Mon Feb 24 12:53:40 2014
@@ -53,12 +53,16 @@ import java.util.zip.GZIPOutputStream;
  * You can change the default maximal number of temporary files with the -t flag: java
  * org/apache/oak/commons/sort/ExternalSort somefile.txt out.txt -t 3
  * 
- * For very large files, you might want to use an appropriate flag to allocate more memory
to the
- * Java VM: java -Xms2G org/apache/oak/commons/sort/ExternalSort somefile.txt out.txt
+ * You can change the default maximum memory available with the -m flag: java
+ * org/apache/oak/commons/sort/ExternalSort somefile.txt out.txt -m 8192
  * 
- * By (in alphabetical order) Philippe Beaudoin, Eleftherios Chetzakis, Jon Elsas, Christan
Grant,
- * Daniel Haran, Daniel Lemire, Sugumaran Harikrishnan, Jerry Yang, First published: April
2010
- * originally posted at http://lemire.me/blog/archives/2010/04/01/external-memory-sorting-in-java/
+ * For very large files, you might want to use an appropriate flag to allocate more memory
to
+ * the Java VM: java -Xms2G org/apache/oak/commons/sort/ExternalSort somefile.txt out.txt
+ * 
+ * By (in alphabetical order) Philippe Beaudoin, Eleftherios Chetzakis, Jon Elsas, Christan
+ * Grant, Daniel Haran, Daniel Lemire, Sugumaran Harikrishnan, Jerry Yang, First published:
+ * April 2010 originally posted at
+ * http://lemire.me/blog/archives/2010/04/01/external-memory-sorting-in-java/
  */
 public class ExternalSort {
 
@@ -74,12 +78,17 @@ public class ExternalSort {
     }
 
     static int DEFAULTMAXTEMPFILES = 1024;
-
+    
+    /**
+    * Defines the default maximum memory to be used while sorting (8 MB)
+    */
+    static long DEFAULT_MAX_MEM_BYTES = 8388608L;
+    
     // we divide the file into small blocks. If the blocks
     // are too small, we shall create too many temporary files.
     // If they are too big, we shall be using too much memory.
     public static long estimateBestSizeOfBlocks(File filetobesorted,
-            int maxtmpfiles) {
+            int maxtmpfiles, long maxMemory) {
         long sizeoffile = filetobesorted.length() * 2;
         /**
          * We multiply by two because later on someone insisted on counting the memory usage
as 2
@@ -93,11 +102,11 @@ public class ExternalSort {
 
         // on the other hand, we don't want to create many temporary
         // files
-        // for naught. If blocksize is smaller than half the free
-        // memory, grow it.
-        long freemem = Runtime.getRuntime().freeMemory();
-        if (blocksize < freemem / 2) {
-            blocksize = freemem / 2;
+        // for naught. If blocksize is less than maximum allowed memory,
+        // scale the blocksize to be equal to the maxMemory parameter
+
+        if (blocksize < maxMemory) {
+            blocksize = maxMemory;
         }
         return blocksize;
     }
@@ -114,7 +123,7 @@ public class ExternalSort {
      */
     public static List<File> sortInBatch(File file)
             throws IOException {
-        return sortInBatch(file, defaultcomparator, DEFAULTMAXTEMPFILES,
+        return sortInBatch(file, defaultcomparator, DEFAULTMAXTEMPFILES, DEFAULT_MAX_MEM_BYTES,
                 Charset.defaultCharset(), null, false);
     }
 
@@ -130,7 +139,7 @@ public class ExternalSort {
      */
     public static List<File> sortInBatch(File file, Comparator<String> cmp)
             throws IOException {
-        return sortInBatch(file, cmp, DEFAULTMAXTEMPFILES,
+        return sortInBatch(file, cmp, DEFAULTMAXTEMPFILES, DEFAULT_MAX_MEM_BYTES,
                 Charset.defaultCharset(), null, false);
     }
 
@@ -148,7 +157,7 @@ public class ExternalSort {
      */
     public static List<File> sortInBatch(File file, Comparator<String> cmp,
             boolean distinct) throws IOException {
-        return sortInBatch(file, cmp, DEFAULTMAXTEMPFILES,
+        return sortInBatch(file, cmp, DEFAULTMAXTEMPFILES, DEFAULT_MAX_MEM_BYTES,
                 Charset.defaultCharset(), null, distinct);
     }
 
@@ -175,14 +184,14 @@ public class ExternalSort {
      * @return a list of temporary flat files
      */
     public static List<File> sortInBatch(File file, Comparator<String> cmp,
-            int maxtmpfiles, Charset cs, File tmpdirectory,
+            int maxtmpfiles, long maxMemory, Charset cs, File tmpdirectory,
             boolean distinct, int numHeader, boolean usegzip)
             throws IOException {
         List<File> files = new ArrayList<File>();
         BufferedReader fbr = new BufferedReader(new InputStreamReader(
                 new FileInputStream(file), cs));
-        long blocksize = estimateBestSizeOfBlocks(file, maxtmpfiles);// in
-                                                                     // bytes
+        long blocksize = estimateBestSizeOfBlocks(file, maxtmpfiles, maxMemory);// in
+                                                                                // bytes
 
         try {
             List<String> tmplist = new ArrayList<String>();
@@ -243,9 +252,9 @@ public class ExternalSort {
      * @return a list of temporary flat files
      */
     public static List<File> sortInBatch(File file, Comparator<String> cmp,
-            int maxtmpfiles, Charset cs, File tmpdirectory, boolean distinct)
+            int maxtmpfiles, long maxMemory, Charset cs, File tmpdirectory, boolean distinct)
             throws IOException {
-        return sortInBatch(file, cmp, maxtmpfiles, cs, tmpdirectory,
+        return sortInBatch(file, cmp, maxtmpfiles, maxMemory, cs, tmpdirectory,
                 distinct, 0, false);
     }
 
@@ -520,6 +529,8 @@ public class ExternalSort {
         System.out
                 .println("-t or --maxtmpfiles (followed by an integer): specify an upper
bound on the number of temporary files");
         System.out
+                .println("-m or --maxmembytes (followed by a long): specify an upper bound
on the memory");
+        System.out
                 .println("-c or --charset (followed by a charset code): specify the character
set to use (for sorting)");
         System.out
                 .println("-z or --gzip: use compression for the temporary files");
@@ -534,6 +545,7 @@ public class ExternalSort {
         boolean verbose = false;
         boolean distinct = false;
         int maxtmpfiles = DEFAULTMAXTEMPFILES;
+        long maxMemory = DEFAULT_MAX_MEM_BYTES;        
         Charset cs = Charset.defaultCharset();
         String inputfile = null, outputfile = null;
         File tempFileStore = null;
@@ -559,6 +571,15 @@ public class ExternalSort {
                     System.err
                             .println("maxtmpfiles should be positive");
                 }
+            } else if ((args[param].equals("-m") || args[param]
+                        .equals("--maxmembytes"))
+                        && args.length > param + 1) {
+                    param++;
+                    maxMemory = Long.parseLong(args[param]);
+                    if (headersize < 0) {
+                        System.err
+                            .println("maxmembytes should be positive");
+                    }
             } else if ((args[param].equals("-c") || args[param]
                     .equals("--charset"))
                     && args.length > param + 1) {
@@ -597,7 +618,7 @@ public class ExternalSort {
         }
         Comparator<String> comparator = defaultcomparator;
         List<File> l = sortInBatch(new File(inputfile), comparator,
-                maxtmpfiles, cs, tempFileStore, distinct, headersize,
+                maxtmpfiles, maxMemory, cs, tempFileStore, distinct, headersize,
                 usegzip);
         if (verbose)
             System.out

Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java?rev=1571268&r1=1571267&r2=1571268&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
(original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
Mon Feb 24 12:53:40 2014
@@ -70,17 +70,20 @@ public class MarkSweepGarbageCollector i
 
     public static final int DEFAULT_BATCH_COUNT = 2048;
 
+    /** The max last modified time of blobs to consider for garbage collection. */
+    private long maxLastModifiedTime;
+
     /** Run concurrently when possible. */
-    boolean runConcurrently = true;
+    private boolean runConcurrently = true;
 
     /** The number of sweeper threads to use. */
-    int numSweepers = 1;
+    private int numSweepers = 1;
 
     /** The node store. */
-    DocumentNodeStore nodeStore;
+    private DocumentNodeStore nodeStore;
     
     /** The garbage collector file state */
-    GarbageCollectorFileState fs;
+    private GarbageCollectorFileState fs;
 
     /** The configured root to store gc process files. */
     private String root = TEMP_DIR;
@@ -89,6 +92,24 @@ public class MarkSweepGarbageCollector i
     private int batchCount = DEFAULT_BATCH_COUNT;
 
     /**
+     * Gets the max last modified time considered for garbage collection.
+     * 
+     * @return the max last modified time
+     */
+    protected long getMaxLastModifiedTime() {
+        return maxLastModifiedTime;
+    }
+
+    /**
+     * Sets the max last modified time considered for garbage collection.
+     * 
+     * @param maxLastModifiedTime the new max last modified time
+     */
+    protected void setMaxLastModifiedTime(long maxLastModifiedTime) {
+        this.maxLastModifiedTime = maxLastModifiedTime;
+    }
+    
+    /**
      * Gets the root.
      * 
      * @return the root
@@ -125,32 +146,27 @@ public class MarkSweepGarbageCollector i
     }
 
     /**
-     * Instantiates a new blob garbage collector.
-     * 
-     * @param nodeStore
-     *            the node store
-     * @param root
-     *            the root
-     * @param batchCount
-     *            the batch count
-     * @param runBackendConcurrently
-     *            - run the backend iterate concurrently
-     * @param maxSweeperThreads
-     *            the max sweeper threads
-     * @throws IOException
-     *             Signals that an I/O exception has occurred.
+     * @param nodeStore the node store
+     * @param root the root
+     * @param batchCount the batch count
+     * @param runBackendConcurrently - run the backend iterate concurrently
+     * @param maxSweeperThreads the max sweeper threads
+     * @param maxLastModifiedTime the max last modified time
+     * @throws IOException Signals that an I/O exception has occurred.
      */
     public void init(
             NodeStore nodeStore,
             String root,
             int batchCount,
             boolean runBackendConcurrently,
-            int maxSweeperThreads)
+            int maxSweeperThreads,
+            long maxLastModifiedTime)
             throws IOException {
         this.batchCount = batchCount;
         this.root = root;
         this.runConcurrently = runBackendConcurrently;
         this.numSweepers = maxSweeperThreads;
+        this.maxLastModifiedTime = maxLastModifiedTime;        
         init(nodeStore);
     }
 
@@ -356,10 +372,10 @@ public class MarkSweepGarbageCollector i
     class Sweeper implements Runnable {
 
         /** The exception queue. */
-        ConcurrentLinkedQueue<String> exceptionQueue;
+        private ConcurrentLinkedQueue<String> exceptionQueue;
 
         /** The ids to sweep. */
-        List<String> ids;
+        private List<String> ids;
 
         /**
          * Instantiates a new sweeper.
@@ -379,7 +395,7 @@ public class MarkSweepGarbageCollector i
             for (String id : ids) {
                 try {
                     boolean deleted = ((GarbageCollectableBlobStore) nodeStore.getBlobStore())
-                            .deleteChunk(id);
+                            .deleteChunk(id, maxLastModifiedTime);
                     if (!deleted) {
                         exceptionQueue.add(id);
                     }
@@ -394,8 +410,6 @@ public class MarkSweepGarbageCollector i
     /**
      * Iterates the complete node tree.
      * 
-     * @param writer
-     *            the writer
      * @return the list
      * @throws Exception
      *             the exception
@@ -446,9 +460,6 @@ public class MarkSweepGarbageCollector i
      * BlobIdRetriever class to retrieve all blob ids.
      */
     class BlobIdRetriever implements Runnable {
-
-        boolean finished;
-
         @Override
         public void run() {
             retrieve();
@@ -465,7 +476,7 @@ public class MarkSweepGarbageCollector i
                 bufferWriter = new BufferedWriter(
                         new FileWriter(fs.getAvailableRefs()));
                 Iterator<String> idsIter = ((GarbageCollectableBlobStore) nodeStore.getBlobStore())
-                        .getAllChunkIds(0);
+                        .getAllChunkIds(maxLastModifiedTime);
                 List<String> ids = Lists.newArrayList();
                 int blobsCount = 0;
                 while (idsIter.hasNext()) {
@@ -483,8 +494,6 @@ public class MarkSweepGarbageCollector i
 
                 // sort the file
                 fs.sort(fs.getAvailableRefs());
-
-                finished = true;
                 LOG.debug("Ending retrieve of all blobs : " + blobsCount);
             } catch (Exception e) {
                 e.printStackTrace();

Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/cloud/CloudBlobStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/cloud/CloudBlobStore.java?rev=1571268&r1=1571267&r2=1571268&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/cloud/CloudBlobStore.java
(original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/cloud/CloudBlobStore.java
Mon Feb 24 12:53:40 2014
@@ -227,23 +227,27 @@ public class CloudBlobStore extends Abst
 }
 
     @Override
-    public boolean deleteChunk(String chunkId) throws Exception {
+    public boolean deleteChunk(String chunkId, long maxLastModifiedTime) throws Exception
{
         Preconditions.checkNotNull(context);
 
         final org.jclouds.blobstore.BlobStore blobStore = context.getBlobStore();
-        blobStore.removeBlob(cloudContainer, chunkId);
-
+        StorageMetadata metadata = blobStore.blobMetadata(cloudContainer, chunkId);
+        if ((maxLastModifiedTime <= 0) 
+                || (metadata.getLastModified().getTime() <= maxLastModifiedTime)) {
+            blobStore.removeBlob(cloudContainer, chunkId);
+            return true;
+        }
         return true;
     }
 
     class CloudStoreIterator implements Iterator<String> {
-        static final int BATCH = 1000;
+        private static final int BATCH = 1000;
 
-        org.jclouds.blobstore.BlobStore store;
-        long maxLastModifiedTime;
+        private org.jclouds.blobstore.BlobStore store;
+        private long maxLastModifiedTime;
 
-        PageSet<? extends StorageMetadata> set;
-        ArrayDeque<String> queue;
+        private PageSet<? extends StorageMetadata> set;
+        private ArrayDeque<String> queue;
 
         public CloudStoreIterator(org.jclouds.blobstore.BlobStore store,
                 long maxLastModifiedTime) {
@@ -255,7 +259,7 @@ public class CloudBlobStore extends Abst
         @Override
         public boolean hasNext() {
             if ((set == null) || (queue == null)) {
-                set = store.list(cloudContainer, maxResults(1000));
+                set = store.list(cloudContainer, maxResults(BATCH));
                 loadElements(set);
             }
 
@@ -278,8 +282,8 @@ public class CloudBlobStore extends Abst
             Iterator<? extends StorageMetadata> iter = set.iterator();
             while (iter.hasNext()) {
                 StorageMetadata metadata = iter.next();
-                if ((maxLastModifiedTime == 0 || maxLastModifiedTime == -1) ||
-                        (metadata.getLastModified().getTime() <= maxLastModifiedTime))
{
+                if ((maxLastModifiedTime <= 0)
+                        || (metadata.getLastModified().getTime() <= maxLastModifiedTime))
{
                     queue.add(metadata.getName());
                 } else {
                     queue.add(metadata.getName());

Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java?rev=1571268&r1=1571267&r2=1571268&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java
(original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java
Mon Feb 24 12:53:40 2014
@@ -522,9 +522,17 @@ public class DataStoreBlobStore implemen
     }
 
     @Override
-    public boolean deleteChunk(String blobId) throws Exception {
-        ((MultiDataStoreAware) dataStore).deleteRecord(new DataIdentifier(blobId));
-        return true;
+    public boolean deleteChunk(String blobId, long maxLastModifiedTime) throws Exception
{
+        if (dataStore instanceof MultiDataStoreAware) {
+            DataIdentifier identifier = new DataIdentifier(blobId);
+            DataRecord dataRecord = dataStore.getRecord(identifier);
+            if ((maxLastModifiedTime <= 0) 
+                    || dataRecord.getLastModified() <= maxLastModifiedTime) {
+                ((MultiDataStoreAware) dataStore).deleteRecord(identifier);
+                return true;
+            }
+        }
+        return false;
     }
 
     @Override

Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/db/DbBlobStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/db/DbBlobStore.java?rev=1571268&r1=1571267&r2=1571268&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/db/DbBlobStore.java
(original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/db/DbBlobStore.java
Mon Feb 24 12:53:40 2014
@@ -215,16 +215,27 @@ public class DbBlobStore extends Abstrac
     }
 
     @Override
-    public boolean deleteChunk(String chunkId) throws Exception {
+    public boolean deleteChunk(String chunkId, long maxLastModifiedTime) throws Exception
{
         Connection conn = cp.getConnection();
         try {
-            PreparedStatement prep = conn
-                    .prepareStatement("delete from datastore_meta where id = ?");
-            ;;;
-            // TODO  and lastMod <= ?
-            ;;;
-            PreparedStatement prepData = conn
-                    .prepareStatement("delete from datastore_data where id = ?");
+            PreparedStatement prep = null;
+            PreparedStatement prepData = null;
+
+            if (maxLastModifiedTime > 0) {
+                prep = conn.prepareStatement(
+                        "delete from datastore_meta where id = ? and lastMod <= ?");
+                prep.setLong(2, maxLastModifiedTime);
+
+                prepData = conn.prepareStatement(
+                        "delete from datastore_data where id = ? and lastMod <= ?");
+                prepData.setLong(2, maxLastModifiedTime);
+            } else {
+                prep = conn.prepareStatement(
+                        "delete from datastore_meta where id = ?");
+
+                prepData = conn.prepareStatement(
+                        "delete from datastore_data where id = ?");
+            }
             prep.setString(1, chunkId);
             prep.execute();
             prepData.setString(1, chunkId);
@@ -245,7 +256,7 @@ public class DbBlobStore extends Abstrac
         final Connection conn = cp.getConnection();
         PreparedStatement prep = null;
 
-        if ((maxLastModifiedTime != 0) && (maxLastModifiedTime != -1)) {
+        if (maxLastModifiedTime > 0) {
             prep = conn.prepareStatement(
                     "select id from datastore_meta where lastMod <= ?");
             prep.setLong(1, maxLastModifiedTime);

Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoBlobStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoBlobStore.java?rev=1571268&r1=1571267&r2=1571268&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoBlobStore.java
(original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoBlobStore.java
Mon Feb 24 12:53:40 2014
@@ -192,12 +192,10 @@ public class MongoBlobStore extends Abst
     }
     
     @Override
-    public boolean deleteChunk(String chunkId) throws Exception {
+    public boolean deleteChunk(String chunkId, long maxLastModifiedTime) throws Exception
{
         DBCollection collection = getBlobCollection();
-        BasicDBObject removeObj = new BasicDBObject();
-        removeObj.append(MongoBlob.KEY_ID, chunkId);
+        WriteResult result = collection.remove(getBlobQuery(chunkId, maxLastModifiedTime));
 
-        WriteResult result = collection.remove(removeObj);
         if (result.getN() == 1) {
             return true;
         }
@@ -222,6 +220,7 @@ public class MongoBlobStore extends Abst
                         .addOption(Bytes.QUERYOPTION_SLAVEOK);
 
         return new AbstractIterator<String>() {
+            @Override
             protected String computeNext() {
                 if (cur.hasNext()) {
                     MongoBlob blob = (MongoBlob) cur.next();

Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBBlobStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBBlobStore.java?rev=1571268&r1=1571267&r2=1571268&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBBlobStore.java
(original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBBlobStore.java
Mon Feb 24 12:53:40 2014
@@ -286,12 +286,25 @@ public class RDBBlobStore extends Abstra
     }
 
     @Override
-    public boolean deleteChunk(String chunkId) throws Exception {
+    public boolean deleteChunk(String chunkId, long maxLastModifiedTime) throws Exception
{
         try {
-            PreparedStatement prep = connection.prepareStatement(
-                    "delete from datastore_meta where id = ?");
-            PreparedStatement prepData = connection.prepareStatement(
-                    "delete from datastore_data where id = ?");
+            PreparedStatement prep = null;
+            PreparedStatement prepData = null;
+
+            if (maxLastModifiedTime > 0) {
+                prep = connection.prepareStatement(
+                        "delete from datastore_meta where id = ? and lastMod <= ?");
+                prep.setLong(2, maxLastModifiedTime);
+
+                prepData = connection.prepareStatement(
+                        "delete from datastore_data where id = ? and lastMod <= ?");
+                prepData.setLong(2, maxLastModifiedTime);
+            } else {
+                prep = connection.prepareStatement(
+                        "delete from datastore_meta where id = ?");
+                prepData = connection.prepareStatement(
+                        "delete from datastore_data where id = ?");
+            }
             prep.setString(1, chunkId);
             prep.execute();
             prepData.setString(1, chunkId);
@@ -301,7 +314,7 @@ public class RDBBlobStore extends Abstra
             prepData.close();
         } finally {
             connection.commit();
-}
+        }
 
         return true;
     }
@@ -310,7 +323,7 @@ public class RDBBlobStore extends Abstra
     public Iterator<String> getAllChunkIds(long maxLastModifiedTime) throws Exception
{
         PreparedStatement prep = null;
 
-        if ((maxLastModifiedTime != 0) && (maxLastModifiedTime != -1)) {
+        if (maxLastModifiedTime > 0) {
             prep = connection.prepareStatement(
                     "select id from datastore_meta where lastMod <= ?");
             prep.setLong(1, maxLastModifiedTime);

Modified: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/MongoBlobGCTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/MongoBlobGCTest.java?rev=1571268&r1=1571267&r2=1571268&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/MongoBlobGCTest.java
(original)
+++ jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/MongoBlobGCTest.java
Mon Feb 24 12:53:40 2014
@@ -94,7 +94,7 @@ public class MongoBlobGCTest extends Abs
 
         DocumentNodeStore s = mk.getNodeStore();
         MarkSweepGarbageCollector gc = new MarkSweepGarbageCollector();
-        gc.init(s, "./target", 2048, true, 2);
+        gc.init(s, "./target", 2048, true, 2, 0);
         gc.collectGarbage();
 
         Set<String> existing = iterate();

Modified: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/blob/AbstractBlobStoreTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/blob/AbstractBlobStoreTest.java?rev=1571268&r1=1571267&r2=1571268&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/blob/AbstractBlobStoreTest.java
(original)
+++ jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/blob/AbstractBlobStoreTest.java
Mon Feb 24 12:53:40 2014
@@ -383,7 +383,7 @@ public abstract class AbstractBlobStoreT
         Set<String> ids = createArtifacts();
 
         for (String id : ids) {
-            store.deleteChunk(id);
+            store.deleteChunk(id, 0);
         }
 
         Iterator<String> iter = store.getAllChunkIds(0);



Mime
View raw message