jackrabbit-oak-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ju...@apache.org
Subject svn commit: r1584152 - in /jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file: FileStore.java TarReader.java TarWriter.java
Date Wed, 02 Apr 2014 20:14:51 GMT
Author: jukka
Date: Wed Apr  2 20:14:50 2014
New Revision: 1584152

URL: http://svn.apache.org/r1584152
Log:
OAK-631: SegmentMK: Implement garbage collection

Activate the FileStore.gc() method

Modified:
    jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java
    jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarReader.java
    jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarWriter.java

Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java?rev=1584152&r1=1584151&r2=1584152&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java
(original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java
Wed Apr  2 20:14:50 2014
@@ -23,6 +23,7 @@ import static com.google.common.collect.
 import static com.google.common.collect.Lists.newLinkedList;
 import static com.google.common.collect.Maps.newHashMap;
 import static com.google.common.collect.Maps.newTreeMap;
+import static com.google.common.collect.Sets.newHashSet;
 import static java.lang.String.format;
 import static java.util.Collections.singletonMap;
 import static java.util.concurrent.TimeUnit.NANOSECONDS;
@@ -34,6 +35,7 @@ import java.io.IOException;
 import java.io.RandomAccessFile;
 import java.nio.ByteBuffer;
 import java.util.Arrays;
+import java.util.Iterator;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
@@ -41,6 +43,7 @@ import java.util.Set;
 import java.util.SortedMap;
 import java.util.UUID;
 import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
@@ -113,6 +116,16 @@ public class FileStore implements Segmen
     private final Thread flushThread;
 
     /**
+     * Flag to request revision cleanup during the next flush.
+     */
+    private final AtomicBoolean cleanupNeeded = new AtomicBoolean(false);
+
+    /**
+     * List of old tar file generations that are waiting to be removed.
+     */
+    private final LinkedList<File> toBeRemoved = newLinkedList();
+
+    /**
      * Synchronization aid used by the background flush thread to stop itself
      * as soon as the {@link #close()} method is called.
      */
@@ -310,6 +323,39 @@ public class FileStore implements Segmen
                     journalFile.writeBytes(after + " root\n");
                     journalFile.getChannel().force(false);
                     persistedHead.set(after);
+
+                    if (cleanupNeeded.getAndSet(false)) {
+                        Set<UUID> ids = newHashSet();
+                        for (SegmentId id : tracker.getReferencedSegmentIds()) {
+                            ids.add(new UUID(
+                                    id.getMostSignificantBits(),
+                                    id.getLeastSignificantBits()));
+                        }
+
+                        List<TarReader> list =
+                                newArrayListWithCapacity(readers.size());
+                        for (TarReader reader : readers) {
+                            TarReader cleaned = reader.cleanup(ids);
+                            if (cleaned == reader) {
+                                list.add(reader);
+                            } else {
+                                if (cleaned != null) {
+                                    list.add(cleaned);
+                                }
+                                toBeRemoved.addLast(reader.close());
+                            }
+                        }
+                        readers = list;
+                    }
+                }
+
+                // remove all obsolete tar generations
+                Iterator<File> iterator = toBeRemoved.iterator();
+                while (iterator.hasNext()) {
+                    File file = iterator.next();
+                    if (!file.exists() || file.delete()) {
+                        iterator.remove();
+                    }
                 }
             }
         }
@@ -476,6 +522,9 @@ public class FileStore implements Segmen
                 list.addAll(readers);
                 readers = list;
 
+                // trigger revision cleanup after next flush
+                cleanupNeeded.set(true);
+
                 writeNumber++;
                 writeFile = new File(
                         directory,
@@ -504,7 +553,6 @@ public class FileStore implements Segmen
     @Override
     public void gc() {
         System.gc();
-        Set<SegmentId> ids = tracker.getReferencedSegmentIds();
-        // TODO reclaim unreferenced segments
+        cleanupNeeded.set(true);
     }
 }
\ No newline at end of file

Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarReader.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarReader.java?rev=1584152&r1=1584151&r2=1584152&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarReader.java
(original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarReader.java
Wed Apr  2 20:14:50 2014
@@ -325,7 +325,10 @@ class TarReader {
         size += getEntrySize(24 * count + 16);
         size += 2 * BLOCK_SIZE;
 
-        if (size >= access.length() * 3 / 4) {
+        if (count == 0) {
+            // none of the entries within this tar file are referenceable
+            return null;
+        } else if (size >= access.length() * 3 / 4) {
             // the space savings are not worth it at less than 25%
             return this;
         }
@@ -356,8 +359,9 @@ class TarReader {
         return new TarReader(newFile, access.isMemoryMapped());
     }
 
-    void close() throws IOException {
+    File close() throws IOException {
         access.close();
+        return file;
     }
 
     //-----------------------------------------------------------< private >--

Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarWriter.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarWriter.java?rev=1584152&r1=1584151&r2=1584152&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarWriter.java
(original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarWriter.java
Wed Apr  2 20:14:50 2014
@@ -28,8 +28,6 @@ import java.io.FileDescriptor;
 import java.io.IOException;
 import java.io.RandomAccessFile;
 import java.nio.ByteBuffer;
-import java.nio.MappedByteBuffer;
-import java.nio.channels.FileChannel.MapMode;
 import java.util.Arrays;
 import java.util.Map;
 import java.util.Set;
@@ -93,8 +91,6 @@ class TarWriter {
      */
     private final Map<UUID, TarEntry> index = newHashMap();
 
-    private MappedByteBuffer buffer = null;
-
     TarWriter(File file) {
         this.file = file;
     }
@@ -112,16 +108,12 @@ class TarWriter {
         checkState(!closed);
         TarEntry entry = index.get(new UUID(msb, lsb));
         if (entry != null) {
-            if (buffer == null
-                    || buffer.remaining() < entry.offset() + entry.size()) {
-                checkState(access != null); // implied by entry != null
-                buffer = access.getChannel().map(
-                        MapMode.READ_ONLY, 0, access.getFilePointer());
-            }
-            ByteBuffer data = buffer.asReadOnlyBuffer();
-            data.position(entry.offset());
-            data.limit(data.position() + entry.size());
-            return data.slice();
+            checkState(access != null); // implied by entry != null
+            ByteBuffer data = ByteBuffer.allocate(entry.size());
+            access.seek(entry.offset());
+            access.readFully(data.array());
+            access.seek(access.length());
+            return data;
         } else {
             return null;
         }



Mime
View raw message