jackrabbit-oak-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ju...@apache.org
Subject svn commit: r1584258 - in /jackrabbit/oak/trunk: oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/ oak-run/src/main/java/org/apache/jackrabbit/oak/run/
Date Thu, 03 Apr 2014 04:47:05 GMT
Author: jukka
Date: Thu Apr  3 04:47:05 2014
New Revision: 1584258

URL: http://svn.apache.org/r1584258
Log:
OAK-631: SegmentMK: Implement garbage collection

Simplify TarWriter.cleanup()
Allow cleanup to proceed even when no commits are pending
Extend the oak-run debug feature to allow inspection of existing data

Modified:
    jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java
    jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarWriter.java
    jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Main.java

Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java?rev=1584258&r1=1584257&r2=1584258&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java
(original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java
Thu Apr  3 04:47:05 2014
@@ -310,7 +310,8 @@ public class FileStore implements Segmen
         synchronized (persistedHead) {
             RecordId before = persistedHead.get();
             RecordId after = head.get();
-            if (!after.equals(before)) {
+            boolean cleanup = cleanupNeeded.getAndSet(false);
+            if (cleanup || !after.equals(before)) {
                 // needs to happen outside the synchronization block below to
                 // avoid a deadlock with another thread flushing the writer
                 tracker.getWriter().flush();
@@ -324,7 +325,7 @@ public class FileStore implements Segmen
                     journalFile.getChannel().force(false);
                     persistedHead.set(after);
 
-                    if (cleanupNeeded.getAndSet(false)) {
+                    if (cleanup) {
                         Set<UUID> ids = newHashSet();
                         for (SegmentId id : tracker.getReferencedSegmentIds()) {
                             ids.add(new UUID(

Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarWriter.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarWriter.java?rev=1584258&r1=1584257&r2=1584258&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarWriter.java
(original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarWriter.java
Thu Apr  3 04:47:05 2014
@@ -93,6 +93,8 @@ class TarWriter {
      */
     private final Map<UUID, TarEntry> index = newHashMap();
 
+    private final Set<UUID> references = newHashSet();
+
     TarWriter(File file) {
         this.file = file;
     }
@@ -159,6 +161,18 @@ class TarWriter {
                 (int) (length - size - padding), size);
         index.put(uuid, entry);
 
+        if (isDataSegmentId(uuid.getLeastSignificantBits())) {
+            ByteBuffer segment = ByteBuffer.wrap(data, offset, size);
+            int pos = segment.position();
+            int refcount = segment.get(pos + REF_COUNT_OFFSET) & 0xff;
+            int refend = pos + 16 * (refcount + 1);
+            for (int refpos = pos + 16; refpos < refend; refpos += 16) {
+                references.add(new UUID(
+                        segment.getLong(refpos),
+                        segment.getLong(refpos + 8)));
+            }
+        }
+
         return length;
     }
 
@@ -303,30 +317,8 @@ class TarWriter {
     }
 
     synchronized void cleanup(Set<UUID> referencedIds) throws IOException {
-        TarEntry[] sorted = index.values().toArray(new TarEntry[index.size()]);
-        Arrays.sort(sorted, TarEntry.OFFSET_ORDER);
-
-        for (int i = sorted.length - 1; i >= 0; i--) {
-            TarEntry entry = sorted[i];
-            UUID id = new UUID(entry.msb(), entry.lsb());
-            if (referencedIds.remove(id) && isDataSegmentId(entry.lsb())) {
-                // this is a referenced data segment, so follow the graph
-                ByteBuffer segment = ByteBuffer.allocate(
-                        Math.min(entry.size(), 16 * 256));
-                access.seek(entry.offset());
-                access.readFully(segment.array());
-                int pos = segment.position();
-                int refcount = segment.get(pos + REF_COUNT_OFFSET) & 0xff;
-                int refend = pos + 16 * (refcount + 1);
-                for (int refpos = pos + 16; refpos < refend; refpos += 16) {
-                    referencedIds.add(new UUID(
-                            segment.getLong(refpos),
-                            segment.getLong(refpos + 8)));
-                }
-            }
-        }
-
-        access.seek(access.length());
+        referencedIds.removeAll(index.keySet());
+        referencedIds.addAll(references);
     }
 
     //------------------------------------------------------------< Object >--

Modified: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Main.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Main.java?rev=1584258&r1=1584257&r2=1584258&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Main.java (original)
+++ jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Main.java Thu
Apr  3 04:47:05 2014
@@ -27,6 +27,8 @@ import java.util.Properties;
 import java.util.Queue;
 import java.util.Set;
 import java.util.UUID;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 import javax.jcr.Repository;
 
@@ -44,16 +46,20 @@ import org.apache.jackrabbit.core.config
 import org.apache.jackrabbit.oak.Oak;
 import org.apache.jackrabbit.oak.api.ContentRepository;
 import org.apache.jackrabbit.oak.benchmark.BenchmarkRunner;
+import org.apache.jackrabbit.oak.commons.PathUtils;
 import org.apache.jackrabbit.oak.fixture.OakFixture;
 import org.apache.jackrabbit.oak.http.OakServlet;
 import org.apache.jackrabbit.oak.jcr.Jcr;
 import org.apache.jackrabbit.oak.plugins.backup.FileStoreBackup;
 import org.apache.jackrabbit.oak.plugins.document.DocumentMK;
 import org.apache.jackrabbit.oak.plugins.document.DocumentNodeStore;
+import org.apache.jackrabbit.oak.plugins.segment.RecordId;
 import org.apache.jackrabbit.oak.plugins.segment.Segment;
 import org.apache.jackrabbit.oak.plugins.segment.SegmentId;
+import org.apache.jackrabbit.oak.plugins.segment.SegmentNodeState;
 import org.apache.jackrabbit.oak.plugins.segment.SegmentNodeStore;
 import org.apache.jackrabbit.oak.plugins.segment.file.FileStore;
+import org.apache.jackrabbit.oak.spi.state.NodeState;
 import org.apache.jackrabbit.oak.spi.state.NodeStore;
 import org.apache.jackrabbit.oak.upgrade.RepositoryUpgrade;
 import org.apache.jackrabbit.server.remoting.davex.JcrRemotingServlet;
@@ -172,6 +178,7 @@ public class Main {
                             dataCount++;
                             dataSize += segment.size();
                             idmap.put(id, segment.getReferencedIds());
+                            System.out.println(id + " -> " + idmap.get(id));
                         } else if (id.isBulkSegmentId()) {
                             bulkCount++;
                             bulkSize += id.getSegment().size();
@@ -216,12 +223,35 @@ public class Main {
                             "%6dMB in %6d bulk segments%n",
                             bulkSize / (1024 * 1024), bulkCount);
                 } else {
+                    Pattern pattern = Pattern.compile(
+                            "([0-9a-f-]+)|([0-9a-f-]+:[0-9a-f]+)?(/.*)?");
                     for (int i = 1; i < args.length; i++) {
-                        UUID uuid = UUID.fromString(args[i]);
-                        SegmentId id = store.getTracker().getSegmentId(
-                                uuid.getMostSignificantBits(),
-                                uuid.getLeastSignificantBits());
-                        System.out.println(id.getSegment());
+                        Matcher matcher = pattern.matcher(args[i]);
+                        if (!matcher.matches()) {
+                            System.err.println("Unknown argument: " + args[i]);
+                        } else if (matcher.group(1) != null) {
+                            UUID uuid = UUID.fromString(matcher.group(1));
+                            SegmentId id = store.getTracker().getSegmentId(
+                                    uuid.getMostSignificantBits(),
+                                    uuid.getLeastSignificantBits());
+                            System.out.println(id.getSegment());
+                        } else {
+                            RecordId id = store.getHead().getRecordId();
+                            if (matcher.group(2) != null) {
+                                id = RecordId.fromString(
+                                        store.getTracker(), matcher.group(2));
+                            }
+                            String path = "/";
+                            if (matcher.group(3) != null) {
+                                path = matcher.group(3);
+                            }
+                            NodeState node = new SegmentNodeState(id);
+                            System.out.println("/ -> " + node);
+                            for (String name : PathUtils.elements(path)) {
+                                node = node.getChildNode(name);
+                                System.out.println(" " + name  + " -> " + node);
+                            }
+                        }
                     }
                 }
             } finally {



Mime
View raw message