Return-Path: X-Original-To: apmail-jackrabbit-oak-commits-archive@minotaur.apache.org Delivered-To: apmail-jackrabbit-oak-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id BDC5E109E5 for ; Wed, 2 Apr 2014 15:18:21 +0000 (UTC) Received: (qmail 50595 invoked by uid 500); 2 Apr 2014 15:18:21 -0000 Delivered-To: apmail-jackrabbit-oak-commits-archive@jackrabbit.apache.org Received: (qmail 50547 invoked by uid 500); 2 Apr 2014 15:18:15 -0000 Mailing-List: contact oak-commits-help@jackrabbit.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: oak-dev@jackrabbit.apache.org Delivered-To: mailing list oak-commits@jackrabbit.apache.org Received: (qmail 50538 invoked by uid 99); 2 Apr 2014 15:18:14 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 02 Apr 2014 15:18:14 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 02 Apr 2014 15:18:11 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id B4C5D238883D; Wed, 2 Apr 2014 15:17:51 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1584067 - in /jackrabbit/oak/trunk: oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/ oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/ oak-run/src/main/java/org/apache/jackrabbit/oak/benchmark/ Date: Wed, 02 Apr 2014 15:17:51 -0000 To: oak-commits@jackrabbit.apache.org From: jukka@apache.org X-Mailer: svnmailer-1.0.9 Message-Id: <20140402151751.B4C5D238883D@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: jukka Date: Wed Apr 2 15:17:50 2014 New Revision: 1584067 URL: http://svn.apache.org/r1584067 Log: OAK-631: SegmentMK: Implement garbage collection Use memory mapping to access already written entries in TarWriter Improve the cache handling in SegmentTracker Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Segment.java jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentId.java jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentTracker.java jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentWriter.java jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarEntry.java jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarWriter.java jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/benchmark/SmallFileReadTest.java Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Segment.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Segment.java?rev=1584067&r1=1584066&r2=1584067&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Segment.java (original) +++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Segment.java Wed Apr 2 15:17:50 2014 @@ -118,6 +118,8 @@ public class Segment { */ private final ConcurrentMap templates = newConcurrentMap(); + private volatile long accessed = 0; + public Segment(SegmentTracker tracker, SegmentId id, ByteBuffer data) { this.tracker = checkNotNull(tracker); this.id = checkNotNull(id); @@ -144,6 +146,15 @@ public class Segment { refids[0] = id; } + void access() { + accessed++; + } + + boolean accessed() { + accessed >>>= 1; + return accessed != 0; + } + /** * Maps the given record offset to the respective position within the * internal {@link #data} array. The validity of a record with the given Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentId.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentId.java?rev=1584067&r1=1584066&r2=1584067&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentId.java (original) +++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentId.java Wed Apr 2 15:17:50 2014 @@ -91,6 +91,7 @@ public class SegmentId implements Compar } } } + segment.access(); return segment; } Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentTracker.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentTracker.java?rev=1584067&r1=1584066&r2=1584067&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentTracker.java (original) +++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentTracker.java Wed Apr 2 15:17:50 2014 @@ -21,6 +21,7 @@ import static com.google.common.collect. import static com.google.common.collect.Sets.newHashSet; import static com.google.common.collect.Sets.newIdentityHashSet; +import java.nio.ByteBuffer; import java.security.SecureRandom; import java.util.LinkedList; import java.util.Queue; @@ -70,9 +71,7 @@ public class SegmentTracker { */ private final SegmentIdTable[] tables = new SegmentIdTable[32]; - private final LinkedList dataSegments = newLinkedList(); - - private final LinkedList bulkSegments = newLinkedList(); + private final LinkedList segments = newLinkedList(); private long currentSize = 0; @@ -105,21 +104,21 @@ public class SegmentTracker { } void setSegment(SegmentId id, Segment segment) { + // done before synchronization to allow concurrent segment access + // while we update the cache below id.setSegment(segment); - LinkedList segments = dataSegments; - if (id.isBulkSegmentId()) { - segments = bulkSegments; - } - synchronized (this) { - // TODO: use a scan-resistant cache segments.addFirst(segment); currentSize += segment.getCacheSize(); - while (currentSize > cacheSize / 2 && segments.size() > 1) { - Segment remove = segments.removeLast(); - remove.getSegmentId().setSegment(null); - currentSize -= remove.getCacheSize(); + while (currentSize > cacheSize && segments.size() > 1) { + Segment last = segments.removeLast(); + if (last.accessed()) { + segments.addFirst(last); + } else { + last.getSegmentId().setSegment(null); + currentSize -= last.getCacheSize(); + } } } } Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentWriter.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentWriter.java?rev=1584067&r1=1584066&r2=1584067&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentWriter.java (original) +++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentWriter.java Wed Apr 2 15:17:50 2014 @@ -205,18 +205,19 @@ public class SegmentWriter { buffer[pos++] = (byte) (offset >> Segment.RECORD_ALIGN_BITS); } - store.writeSegment( - segment.getSegmentId(), - buffer, buffer.length - length, length); - - if (length < buffer.length / 2) { - byte[] data = new byte[length]; - System.arraycopy( - buffer, buffer.length - length, data, 0, length); - segment = new Segment( - tracker, segment.getSegmentId(), ByteBuffer.wrap(data)); + SegmentId id = segment.getSegmentId(); + store.writeSegment(id, buffer, buffer.length - length, length); + + // Keep this segment in memory as it's likely to be accessed soon + ByteBuffer data; + if (buffer.length - length > 4096) { + data = ByteBuffer.allocate(length); + data.put(buffer, buffer.length - length, length); + data.rewind(); + } else { + data = ByteBuffer.wrap(buffer); } - tracker.setSegment(segment.getSegmentId(), segment); + tracker.setSegment(id, new Segment(tracker, id, data)); buffer = createNewBuffer(); roots.clear(); Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java?rev=1584067&r1=1584066&r2=1584067&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java (original) +++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java Wed Apr 2 15:17:50 2014 @@ -433,9 +433,13 @@ public class FileStore implements Segmen } synchronized (this) { - ByteBuffer buffer = writer.readEntry(msb, lsb); - if (buffer != null) { - return new Segment(tracker, id, buffer); + try { + ByteBuffer buffer = writer.readEntry(msb, lsb); + if (buffer != null) { + return new Segment(tracker, id, buffer); + } + } catch (IOException e) { + log.warn("Failed to read from tar file " + writer, e); } } Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarEntry.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarEntry.java?rev=1584067&r1=1584066&r2=1584067&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarEntry.java (original) +++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarEntry.java Wed Apr 2 15:17:50 2014 @@ -81,4 +81,4 @@ class TarEntry { return size; } -} \ No newline at end of file +} Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarWriter.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarWriter.java?rev=1584067&r1=1584066&r2=1584067&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarWriter.java (original) +++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarWriter.java Wed Apr 2 15:17:50 2014 @@ -28,6 +28,8 @@ import java.io.FileDescriptor; import java.io.IOException; import java.io.RandomAccessFile; import java.nio.ByteBuffer; +import java.nio.MappedByteBuffer; +import java.nio.channels.FileChannel.MapMode; import java.util.Arrays; import java.util.Map; import java.util.Set; @@ -91,6 +93,8 @@ class TarWriter { */ private final Map index = newHashMap(); + private MappedByteBuffer buffer = null; + TarWriter(File file) { this.file = file; } @@ -104,24 +108,20 @@ class TarWriter { return index.containsKey(new UUID(msb, lsb)); } - synchronized ByteBuffer readEntry(long msb, long lsb) { + synchronized ByteBuffer readEntry(long msb, long lsb) throws IOException { checkState(!closed); TarEntry entry = index.get(new UUID(msb, lsb)); if (entry != null) { - checkState(access != null); // implied by entry != null - try { - try { - byte[] data = new byte[entry.size()]; - access.seek(entry.offset()); - access.readFully(data); - return ByteBuffer.wrap(data); - } finally { - access.seek(access.length()); - } - } catch (IOException e) { - throw new RuntimeException( - "Unable to read from tar file " + file, e); + if (buffer == null + || buffer.remaining() < entry.offset() + entry.size()) { + checkState(access != null); // implied by entry != null + buffer = access.getChannel().map( + MapMode.READ_ONLY, 0, access.getFilePointer()); } + ByteBuffer data = buffer.asReadOnlyBuffer(); + data.position(entry.offset()); + data.limit(data.position() + entry.size()); + return data.slice(); } else { return null; } @@ -249,6 +249,7 @@ class TarWriter { access.write(ZERO_BYTES); access.write(ZERO_BYTES); access.close(); + buffer = null; } } Modified: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/benchmark/SmallFileReadTest.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/benchmark/SmallFileReadTest.java?rev=1584067&r1=1584066&r2=1584067&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/benchmark/SmallFileReadTest.java (original) +++ jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/benchmark/SmallFileReadTest.java Wed Apr 2 15:17:50 2014 @@ -29,7 +29,7 @@ public class SmallFileReadTest extends A private static final int FILE_COUNT = 1000; - private static final int FILE_SIZE = 10; + private static final int FILE_SIZE = Integer.getInteger("file.size", 10); private Session session;