Return-Path: X-Original-To: apmail-jackrabbit-oak-commits-archive@minotaur.apache.org Delivered-To: apmail-jackrabbit-oak-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id AB62F10877 for ; Wed, 2 Apr 2014 20:15:25 +0000 (UTC) Received: (qmail 4036 invoked by uid 500); 2 Apr 2014 20:15:25 -0000 Delivered-To: apmail-jackrabbit-oak-commits-archive@jackrabbit.apache.org Received: (qmail 3500 invoked by uid 500); 2 Apr 2014 20:15:18 -0000 Mailing-List: contact oak-commits-help@jackrabbit.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: oak-dev@jackrabbit.apache.org Delivered-To: mailing list oak-commits@jackrabbit.apache.org Received: (qmail 3462 invoked by uid 99); 2 Apr 2014 20:15:13 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 02 Apr 2014 20:15:13 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 02 Apr 2014 20:15:11 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 475F423889E1; Wed, 2 Apr 2014 20:14:51 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1584152 - in /jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file: FileStore.java TarReader.java TarWriter.java Date: Wed, 02 Apr 2014 20:14:51 -0000 To: oak-commits@jackrabbit.apache.org From: jukka@apache.org X-Mailer: svnmailer-1.0.9 Message-Id: <20140402201451.475F423889E1@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: jukka Date: Wed Apr 2 20:14:50 2014 New Revision: 1584152 URL: http://svn.apache.org/r1584152 Log: OAK-631: SegmentMK: Implement garbage collection Activate the FileStore.gc() method Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarReader.java jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarWriter.java Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java?rev=1584152&r1=1584151&r2=1584152&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java (original) +++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java Wed Apr 2 20:14:50 2014 @@ -23,6 +23,7 @@ import static com.google.common.collect. import static com.google.common.collect.Lists.newLinkedList; import static com.google.common.collect.Maps.newHashMap; import static com.google.common.collect.Maps.newTreeMap; +import static com.google.common.collect.Sets.newHashSet; import static java.lang.String.format; import static java.util.Collections.singletonMap; import static java.util.concurrent.TimeUnit.NANOSECONDS; @@ -34,6 +35,7 @@ import java.io.IOException; import java.io.RandomAccessFile; import java.nio.ByteBuffer; import java.util.Arrays; +import java.util.Iterator; import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -41,6 +43,7 @@ import java.util.Set; import java.util.SortedMap; import java.util.UUID; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -113,6 +116,16 @@ public class FileStore implements Segmen private final Thread flushThread; /** + * Flag to request revision cleanup during the next flush. + */ + private final AtomicBoolean cleanupNeeded = new AtomicBoolean(false); + + /** + * List of old tar file generations that are waiting to be removed. + */ + private final LinkedList toBeRemoved = newLinkedList(); + + /** * Synchronization aid used by the background flush thread to stop itself * as soon as the {@link #close()} method is called. */ @@ -310,6 +323,39 @@ public class FileStore implements Segmen journalFile.writeBytes(after + " root\n"); journalFile.getChannel().force(false); persistedHead.set(after); + + if (cleanupNeeded.getAndSet(false)) { + Set ids = newHashSet(); + for (SegmentId id : tracker.getReferencedSegmentIds()) { + ids.add(new UUID( + id.getMostSignificantBits(), + id.getLeastSignificantBits())); + } + + List list = + newArrayListWithCapacity(readers.size()); + for (TarReader reader : readers) { + TarReader cleaned = reader.cleanup(ids); + if (cleaned == reader) { + list.add(reader); + } else { + if (cleaned != null) { + list.add(cleaned); + } + toBeRemoved.addLast(reader.close()); + } + } + readers = list; + } + } + + // remove all obsolete tar generations + Iterator iterator = toBeRemoved.iterator(); + while (iterator.hasNext()) { + File file = iterator.next(); + if (!file.exists() || file.delete()) { + iterator.remove(); + } } } } @@ -476,6 +522,9 @@ public class FileStore implements Segmen list.addAll(readers); readers = list; + // trigger revision cleanup after next flush + cleanupNeeded.set(true); + writeNumber++; writeFile = new File( directory, @@ -504,7 +553,6 @@ public class FileStore implements Segmen @Override public void gc() { System.gc(); - Set ids = tracker.getReferencedSegmentIds(); - // TODO reclaim unreferenced segments + cleanupNeeded.set(true); } } \ No newline at end of file Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarReader.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarReader.java?rev=1584152&r1=1584151&r2=1584152&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarReader.java (original) +++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarReader.java Wed Apr 2 20:14:50 2014 @@ -325,7 +325,10 @@ class TarReader { size += getEntrySize(24 * count + 16); size += 2 * BLOCK_SIZE; - if (size >= access.length() * 3 / 4) { + if (count == 0) { + // none of the entries within this tar file are referenceable + return null; + } else if (size >= access.length() * 3 / 4) { // the space savings are not worth it at less than 25% return this; } @@ -356,8 +359,9 @@ class TarReader { return new TarReader(newFile, access.isMemoryMapped()); } - void close() throws IOException { + File close() throws IOException { access.close(); + return file; } //-----------------------------------------------------------< private >-- Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarWriter.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarWriter.java?rev=1584152&r1=1584151&r2=1584152&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarWriter.java (original) +++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarWriter.java Wed Apr 2 20:14:50 2014 @@ -28,8 +28,6 @@ import java.io.FileDescriptor; import java.io.IOException; import java.io.RandomAccessFile; import java.nio.ByteBuffer; -import java.nio.MappedByteBuffer; -import java.nio.channels.FileChannel.MapMode; import java.util.Arrays; import java.util.Map; import java.util.Set; @@ -93,8 +91,6 @@ class TarWriter { */ private final Map index = newHashMap(); - private MappedByteBuffer buffer = null; - TarWriter(File file) { this.file = file; } @@ -112,16 +108,12 @@ class TarWriter { checkState(!closed); TarEntry entry = index.get(new UUID(msb, lsb)); if (entry != null) { - if (buffer == null - || buffer.remaining() < entry.offset() + entry.size()) { - checkState(access != null); // implied by entry != null - buffer = access.getChannel().map( - MapMode.READ_ONLY, 0, access.getFilePointer()); - } - ByteBuffer data = buffer.asReadOnlyBuffer(); - data.position(entry.offset()); - data.limit(data.position() + entry.size()); - return data.slice(); + checkState(access != null); // implied by entry != null + ByteBuffer data = ByteBuffer.allocate(entry.size()); + access.seek(entry.offset()); + access.readFully(data.array()); + access.seek(access.length()); + return data; } else { return null; }