jackrabbit-oak-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ju...@apache.org
Subject svn commit: r1584067 - in /jackrabbit/oak/trunk: oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/ oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/ oak-run/src/main/java/org/apache/jackrabbit/oak/benchmark/
Date Wed, 02 Apr 2014 15:17:51 GMT
Author: jukka
Date: Wed Apr  2 15:17:50 2014
New Revision: 1584067

URL: http://svn.apache.org/r1584067
Log:
OAK-631: SegmentMK: Implement garbage collection

Use memory mapping to access already written entries in TarWriter
Improve the cache handling in SegmentTracker

Modified:
    jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Segment.java
    jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentId.java
    jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentTracker.java
    jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentWriter.java
    jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java
    jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarEntry.java
    jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarWriter.java
    jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/benchmark/SmallFileReadTest.java

Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Segment.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Segment.java?rev=1584067&r1=1584066&r2=1584067&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Segment.java
(original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Segment.java
Wed Apr  2 15:17:50 2014
@@ -118,6 +118,8 @@ public class Segment {
      */
     private final ConcurrentMap<Integer, Template> templates = newConcurrentMap();
 
+    private volatile long accessed = 0;
+
     public Segment(SegmentTracker tracker, SegmentId id, ByteBuffer data) {
         this.tracker = checkNotNull(tracker);
         this.id = checkNotNull(id);
@@ -144,6 +146,15 @@ public class Segment {
         refids[0] = id;
     }
 
+    void access() {
+        accessed++;
+    }
+
+    boolean accessed() {
+        accessed >>>= 1;
+        return accessed != 0;
+    }
+
     /**
      * Maps the given record offset to the respective position within the
      * internal {@link #data} array. The validity of a record with the given

Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentId.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentId.java?rev=1584067&r1=1584066&r2=1584067&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentId.java
(original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentId.java
Wed Apr  2 15:17:50 2014
@@ -91,6 +91,7 @@ public class SegmentId implements Compar
                 }
             }
         }
+        segment.access();
         return segment;
     }
 

Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentTracker.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentTracker.java?rev=1584067&r1=1584066&r2=1584067&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentTracker.java
(original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentTracker.java
Wed Apr  2 15:17:50 2014
@@ -21,6 +21,7 @@ import static com.google.common.collect.
 import static com.google.common.collect.Sets.newHashSet;
 import static com.google.common.collect.Sets.newIdentityHashSet;
 
+import java.nio.ByteBuffer;
 import java.security.SecureRandom;
 import java.util.LinkedList;
 import java.util.Queue;
@@ -70,9 +71,7 @@ public class SegmentTracker {
      */
     private final SegmentIdTable[] tables = new SegmentIdTable[32];
 
-    private final LinkedList<Segment> dataSegments = newLinkedList();
-
-    private final LinkedList<Segment> bulkSegments = newLinkedList();
+    private final LinkedList<Segment> segments = newLinkedList();
 
     private long currentSize = 0;
 
@@ -105,21 +104,21 @@ public class SegmentTracker {
     }
 
     void setSegment(SegmentId id, Segment segment) {
+        // done before synchronization to allow concurrent segment access
+        // while we update the cache below
         id.setSegment(segment);
 
-        LinkedList<Segment> segments = dataSegments;
-        if (id.isBulkSegmentId()) {
-            segments = bulkSegments;
-        }
-
         synchronized (this) {
-            // TODO: use a scan-resistant cache
             segments.addFirst(segment);
             currentSize += segment.getCacheSize();
-            while (currentSize > cacheSize / 2 && segments.size() > 1) {
-                Segment remove = segments.removeLast();
-                remove.getSegmentId().setSegment(null);
-                currentSize -= remove.getCacheSize();
+            while (currentSize > cacheSize && segments.size() > 1) {
+                Segment last = segments.removeLast();
+                if (last.accessed()) {
+                    segments.addFirst(last);
+                } else {
+                    last.getSegmentId().setSegment(null);
+                    currentSize -= last.getCacheSize();
+                }
             }
         }
     }

Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentWriter.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentWriter.java?rev=1584067&r1=1584066&r2=1584067&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentWriter.java
(original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentWriter.java
Wed Apr  2 15:17:50 2014
@@ -205,18 +205,19 @@ public class SegmentWriter {
                 buffer[pos++] = (byte) (offset >> Segment.RECORD_ALIGN_BITS);
             }
 
-            store.writeSegment(
-                    segment.getSegmentId(),
-                    buffer, buffer.length - length, length);
-
-            if (length < buffer.length / 2) {
-                byte[] data = new byte[length];
-                System.arraycopy(
-                        buffer, buffer.length - length, data, 0, length);
-                segment = new Segment(
-                        tracker, segment.getSegmentId(), ByteBuffer.wrap(data));
+            SegmentId id = segment.getSegmentId();
+            store.writeSegment(id, buffer, buffer.length - length, length);
+
+            // Keep this segment in memory as it's likely to be accessed soon
+            ByteBuffer data;
+            if (buffer.length - length > 4096) {
+                data = ByteBuffer.allocate(length);
+                data.put(buffer, buffer.length - length, length);
+                data.rewind();
+            } else {
+                data = ByteBuffer.wrap(buffer);
             }
-            tracker.setSegment(segment.getSegmentId(), segment);
+            tracker.setSegment(id, new Segment(tracker, id, data));
 
             buffer = createNewBuffer();
             roots.clear();

Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java?rev=1584067&r1=1584066&r2=1584067&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java
(original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java
Wed Apr  2 15:17:50 2014
@@ -433,9 +433,13 @@ public class FileStore implements Segmen
         }
 
         synchronized (this) {
-            ByteBuffer buffer = writer.readEntry(msb, lsb);
-            if (buffer != null) {
-                return new Segment(tracker, id, buffer);
+            try {
+                ByteBuffer buffer = writer.readEntry(msb, lsb);
+                if (buffer != null) {
+                    return new Segment(tracker, id, buffer);
+                }
+            } catch (IOException e) {
+                log.warn("Failed to read from tar file " + writer, e);
             }
         }
 

Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarEntry.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarEntry.java?rev=1584067&r1=1584066&r2=1584067&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarEntry.java
(original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarEntry.java
Wed Apr  2 15:17:50 2014
@@ -81,4 +81,4 @@ class TarEntry {
         return size;
     }
 
-}
\ No newline at end of file
+}

Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarWriter.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarWriter.java?rev=1584067&r1=1584066&r2=1584067&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarWriter.java
(original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarWriter.java
Wed Apr  2 15:17:50 2014
@@ -28,6 +28,8 @@ import java.io.FileDescriptor;
 import java.io.IOException;
 import java.io.RandomAccessFile;
 import java.nio.ByteBuffer;
+import java.nio.MappedByteBuffer;
+import java.nio.channels.FileChannel.MapMode;
 import java.util.Arrays;
 import java.util.Map;
 import java.util.Set;
@@ -91,6 +93,8 @@ class TarWriter {
      */
     private final Map<UUID, TarEntry> index = newHashMap();
 
+    private MappedByteBuffer buffer = null;
+
     TarWriter(File file) {
         this.file = file;
     }
@@ -104,24 +108,20 @@ class TarWriter {
         return index.containsKey(new UUID(msb, lsb));
     }
 
-    synchronized ByteBuffer readEntry(long msb, long lsb) {
+    synchronized ByteBuffer readEntry(long msb, long lsb) throws IOException {
         checkState(!closed);
         TarEntry entry = index.get(new UUID(msb, lsb));
         if (entry != null) {
-            checkState(access != null); // implied by entry != null
-            try {
-                try {
-                    byte[] data = new byte[entry.size()];
-                    access.seek(entry.offset());
-                    access.readFully(data);
-                    return ByteBuffer.wrap(data);
-                } finally {
-                    access.seek(access.length());
-                }
-            } catch (IOException e) {
-                throw new RuntimeException(
-                        "Unable to read from tar file " + file, e);
+            if (buffer == null
+                    || buffer.remaining() < entry.offset() + entry.size()) {
+                checkState(access != null); // implied by entry != null
+                buffer = access.getChannel().map(
+                        MapMode.READ_ONLY, 0, access.getFilePointer());
             }
+            ByteBuffer data = buffer.asReadOnlyBuffer();
+            data.position(entry.offset());
+            data.limit(data.position() + entry.size());
+            return data.slice();
         } else {
             return null;
         }
@@ -249,6 +249,7 @@ class TarWriter {
             access.write(ZERO_BYTES);
             access.write(ZERO_BYTES);
             access.close();
+            buffer = null;
         }
     }
 

Modified: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/benchmark/SmallFileReadTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/benchmark/SmallFileReadTest.java?rev=1584067&r1=1584066&r2=1584067&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/benchmark/SmallFileReadTest.java
(original)
+++ jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/benchmark/SmallFileReadTest.java
Wed Apr  2 15:17:50 2014
@@ -29,7 +29,7 @@ public class SmallFileReadTest extends A
 
     private static final int FILE_COUNT = 1000;
 
-    private static final int FILE_SIZE = 10;
+    private static final int FILE_SIZE = Integer.getInteger("file.size", 10);
 
     private Session session;
 



Mime
View raw message