jackrabbit-oak-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ju...@apache.org
Subject svn commit: r1584257 - in /jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment: SegmentNodeStore.java file/FileStore.java file/TarWriter.java
Date Thu, 03 Apr 2014 04:27:58 GMT
Author: jukka
Date: Thu Apr  3 04:27:57 2014
New Revision: 1584257

URL: http://svn.apache.org/r1584257
Log:
OAK-631: SegmentMK: Implement garbage collection

Remember to follow also the reference graph of segments within the TarWriter
Allow old checkpoints to expire

Modified:
    jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentNodeStore.java
    jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java
    jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarWriter.java

Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentNodeStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentNodeStore.java?rev=1584257&r1=1584256&r2=1584257&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentNodeStore.java
(original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentNodeStore.java
Thu Apr  3 04:27:57 2014
@@ -38,6 +38,8 @@ import javax.annotation.Nullable;
 
 import org.apache.jackrabbit.oak.api.Blob;
 import org.apache.jackrabbit.oak.api.CommitFailedException;
+import org.apache.jackrabbit.oak.api.PropertyState;
+import org.apache.jackrabbit.oak.api.Type;
 import org.apache.jackrabbit.oak.plugins.segment.memory.MemoryStore;
 import org.apache.jackrabbit.oak.spi.blob.BlobStore;
 import org.apache.jackrabbit.oak.spi.commit.ChangeDispatcher;
@@ -197,6 +199,7 @@ public class SegmentNodeStore implements
     public synchronized String checkpoint(long lifetime) {
         checkArgument(lifetime > 0);
         String name = UUID.randomUUID().toString();
+        long now = System.currentTimeMillis();
 
         // try 5 times
         for (int i = 0; i < 5; i++) {
@@ -205,11 +208,21 @@ public class SegmentNodeStore implements
                     refreshHead();
 
                     SegmentNodeState state = head.get();
-
                     SegmentNodeBuilder builder = state.builder();
-                    NodeBuilder cp = builder.child(name);
-                    cp.setProperty("timestamp", System.currentTimeMillis()
-                            + lifetime);
+
+                    NodeBuilder checkpoints = builder.child("checkpoints");
+                    for (String n : checkpoints.getChildNodeNames()) {
+                        NodeBuilder cp = checkpoints.getChildNode(n);
+                        PropertyState ts = cp.getProperty("timestamp");
+                        if (ts == null
+                                || ts.getType() != Type.LONG
+                                || now > ts.getValue(Type.LONG)) {
+                            cp.remove();
+                        }
+                    }
+
+                    NodeBuilder cp = checkpoints.child(name);
+                    cp.setProperty("timestamp",  now + lifetime);
                     cp.setChildNode(ROOT, state.getChildNode(ROOT));
 
                     if (store.setHead(state, builder.getNodeState())) {
@@ -228,7 +241,10 @@ public class SegmentNodeStore implements
 
     @Override @CheckForNull
     public NodeState retrieve(@Nonnull String checkpoint) {
-        NodeState cp = head.get().getChildNode(checkpoint).getChildNode(ROOT);
+        NodeState cp = head.get()
+                .getChildNode("checkpoints")
+                .getChildNode(checkpoint)
+                .getChildNode(ROOT);
         if (cp.exists()) {
             return cp;
         }

Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java?rev=1584257&r1=1584256&r2=1584257&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java
(original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java
Thu Apr  3 04:27:57 2014
@@ -331,6 +331,7 @@ public class FileStore implements Segmen
                                     id.getMostSignificantBits(),
                                     id.getLeastSignificantBits()));
                         }
+                        writer.cleanup(ids);
 
                         List<TarReader> list =
                                 newArrayListWithCapacity(readers.size());

Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarWriter.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarWriter.java?rev=1584257&r1=1584256&r2=1584257&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarWriter.java
(original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarWriter.java
Thu Apr  3 04:27:57 2014
@@ -22,6 +22,8 @@ import static com.google.common.base.Pre
 import static com.google.common.base.Preconditions.checkState;
 import static com.google.common.collect.Maps.newHashMap;
 import static com.google.common.collect.Sets.newHashSet;
+import static org.apache.jackrabbit.oak.plugins.segment.Segment.REF_COUNT_OFFSET;
+import static org.apache.jackrabbit.oak.plugins.segment.SegmentId.isDataSegmentId;
 
 import java.io.File;
 import java.io.FileDescriptor;
@@ -300,6 +302,33 @@ class TarWriter {
         return header;
     }
 
+    synchronized void cleanup(Set<UUID> referencedIds) throws IOException {
+        TarEntry[] sorted = index.values().toArray(new TarEntry[index.size()]);
+        Arrays.sort(sorted, TarEntry.OFFSET_ORDER);
+
+        for (int i = sorted.length - 1; i >= 0; i--) {
+            TarEntry entry = sorted[i];
+            UUID id = new UUID(entry.msb(), entry.lsb());
+            if (referencedIds.remove(id) && isDataSegmentId(entry.lsb())) {
+                // this is a referenced data segment, so follow the graph
+                ByteBuffer segment = ByteBuffer.allocate(
+                        Math.min(entry.size(), 16 * 256));
+                access.seek(entry.offset());
+                access.readFully(segment.array());
+                int pos = segment.position();
+                int refcount = segment.get(pos + REF_COUNT_OFFSET) & 0xff;
+                int refend = pos + 16 * (refcount + 1);
+                for (int refpos = pos + 16; refpos < refend; refpos += 16) {
+                    referencedIds.add(new UUID(
+                            segment.getLong(refpos),
+                            segment.getLong(refpos + 8)));
+                }
+            }
+        }
+
+        access.seek(access.length());
+    }
+
     //------------------------------------------------------------< Object >--
 
     @Override



Mime
View raw message