jackrabbit-oak-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ju...@apache.org
Subject svn commit: r1478277 - in /jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment: SegmentNodeStoreService.java SegmentStore.java file/FileStore.java memory/MemoryStore.java mongo/MongoStore.java
Date Thu, 02 May 2013 06:04:08 GMT
Author: jukka
Date: Thu May  2 06:04:08 2013
New Revision: 1478277

URL: http://svn.apache.org/r1478277
Log:
OAK-788: File backend for the SegmentMK

Add support for more than one tar file.
Use proper tar formatting.

Modified:
    jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentNodeStoreService.java
    jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentStore.java
    jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java
    jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/memory/MemoryStore.java
    jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/mongo/MongoStore.java

Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentNodeStoreService.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentNodeStoreService.java?rev=1478277&r1=1478276&r2=1478277&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentNodeStoreService.java
(original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentNodeStoreService.java
Thu May  2 06:04:08 2013
@@ -16,8 +16,7 @@
  */
 package org.apache.jackrabbit.oak.plugins.segment;
 
-import java.io.File;
-import java.net.UnknownHostException;
+import java.io.IOException;
 import java.util.Collection;
 import java.util.Dictionary;
 import java.util.Map;
@@ -69,6 +68,10 @@ public class SegmentNodeStoreService ext
     public SegmentNodeStoreService(final SegmentStore[] store) {
         super(new SegmentStore() {
             @Override
+            public void close() {
+                store[0].close();
+            }
+            @Override
             public Journal getJournal(final String name) {
                 return new Journal() {
                     @Override
@@ -117,16 +120,15 @@ public class SegmentNodeStoreService ext
     }
 
     @Activate
-    public void activate(ComponentContext context) throws UnknownHostException {
+    public void activate(ComponentContext context) throws IOException {
         Dictionary<?, ?> properties = context.getProperties();
         name = "" + properties.get(NAME);
 
         if (properties.get(DIRECTORY) != null) {
-            File directory = new File(properties.get(DIRECTORY).toString());
-            directory.mkdirs();
+            String directory = properties.get(DIRECTORY).toString();
 
             mongo = null;
-            store[0] = new FileStore(new File(directory, "data.tar").getPath());
+            store[0] = new FileStore(directory);
         } else {
             String host = String.valueOf(properties.get(HOST));
             int port = Integer.parseInt(String.valueOf(properties.get(PORT)));
@@ -140,6 +142,7 @@ public class SegmentNodeStoreService ext
 
     @Deactivate
     public void deactivate() {
+        store[0].close();
         if (mongo != null) {
             mongo.close();
         }

Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentStore.java?rev=1478277&r1=1478276&r2=1478277&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentStore.java
(original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentStore.java
Thu May  2 06:04:08 2013
@@ -33,4 +33,6 @@ public interface SegmentStore {
 
     void deleteSegment(UUID segmentId);
 
+    void close();
+
 }

Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java?rev=1478277&r1=1478276&r2=1478277&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java
(original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java
Thu May  2 06:04:08 2013
@@ -17,13 +17,19 @@
 package org.apache.jackrabbit.oak.plugins.segment.file;
 
 import static com.google.common.base.Charsets.UTF_8;
+import static com.google.common.base.Preconditions.checkNotNull;
+import static com.google.common.base.Preconditions.checkState;
+import static com.google.common.collect.Lists.newArrayListWithCapacity;
 import static java.nio.channels.FileChannel.MapMode.READ_WRITE;
 import static org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState.EMPTY_NODE;
 
+import java.io.File;
 import java.io.IOException;
 import java.io.RandomAccessFile;
 import java.nio.ByteBuffer;
+import java.nio.MappedByteBuffer;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.Map;
 import java.util.UUID;
 import java.util.concurrent.ConcurrentMap;
@@ -40,38 +46,99 @@ import com.google.common.collect.Maps;
 
 public class FileStore implements SegmentStore {
 
+    private static final long MAGIC_BYTES = 0x4f616b0a527845ddL;
+
+    private static final long FILE_SIZE = 256 * 1024 * 1024;
+
+    private static final String FILE_NAME_FORMAT = "data%05d.tar";
+
     private final Map<String, Journal> journals = Maps.newHashMap();
 
-    private final ByteBuffer tar;
+    private final File directory;
+    private int index;
+
+    private MappedByteBuffer rw;
+    private ByteBuffer ro;
 
     private final ConcurrentMap<UUID, Segment> segments =
             Maps.newConcurrentMap();
 
-    public FileStore(String filename, NodeState root) {
-        try {
-            RandomAccessFile file = new RandomAccessFile(filename, "rw");
-            try {
-                tar = file.getChannel().map(READ_WRITE, 0, 1024 * 1024 * 1024);
-            } finally {
-                file.close();
-            }
-        } catch (IOException e) {
-            throw new RuntimeException(e);
+    public FileStore(File directory, NodeState root) throws IOException {
+        checkNotNull(directory).mkdirs();
+        this.directory = directory;
+        this.index = 0;
+        while (loadSegments()) {
+            this.index++;
         }
 
         journals.put("root", new MemoryJournal(this, root));
     }
 
-    public FileStore(NodeState root) {
-        this("data.tar", root);
+    public FileStore(File directory) throws IOException {
+        this(directory, EMPTY_NODE);
     }
 
-    public FileStore(String filename) {
-        this(filename, EMPTY_NODE);
+    public FileStore(String directory) throws IOException {
+        this(new File(directory));
     }
 
-    public FileStore() {
-        this(EMPTY_NODE);
+    public void close() {
+        rw.force();
+    }
+
+    private boolean loadSegments() throws IOException {
+        String name = String.format(FILE_NAME_FORMAT, index);
+        File file = new File(directory, name);
+        long size = FILE_SIZE;
+        if (file.isFile()) {
+            size = file.length();
+        }
+        RandomAccessFile f = new RandomAccessFile(file, "rw");
+        try {
+            rw = f.getChannel().map(READ_WRITE, 0, size);
+            ro = rw.asReadOnlyBuffer();
+
+            while (ro.remaining() >= 4 * 0x200) {
+                // skip tar header and get the magic bytes; TODO: verify?
+                long magic = ro.getLong(ro.position() + 0x200);
+                if (magic == MAGIC_BYTES) {
+                    ro.position(ro.position() + 0x200 + 8);
+
+                    int length = ro.getInt();
+                    int count = ro.getInt();
+
+                    UUID segmentId = new UUID(ro.getLong(), ro.getLong());
+                    Collection<UUID> referencedSegmentIds =
+                            newArrayListWithCapacity(count);
+                    for (int i = 0; i < count; i++) {
+                        referencedSegmentIds.add(
+                                new UUID(ro.getLong(), ro.getLong()));
+                    }
+
+                    ro.limit(ro.position() + length);
+                    ByteBuffer data = ro.slice();
+                    ro.limit(rw.limit());
+
+                    Segment segment = new Segment(
+                            this, segmentId, data, referencedSegmentIds,
+                            Collections.<String, RecordId>emptyMap(),
+                            Collections.<Template, RecordId>emptyMap());
+                    segments.put(segmentId, segment);
+
+                    // advance to next entry in the file
+                    ro.position((ro.position() + length + 0x1ff) & ~0x1ff);
+                } else {
+                    // still space for more segments: position the write
+                    // buffer at this point and return false to stop looking
+                    // for more entries
+                    rw.position(ro.position());
+                    return false;
+                }
+            }
+            return true;
+        } finally {
+            f.close();
+        }
     }
 
     @Override
@@ -95,32 +162,54 @@ public class FileStore implements Segmen
     }
 
     @Override
-    public void createSegment(
+    public synchronized void createSegment(
             UUID segmentId, byte[] data, int offset, int length,
             Collection<UUID> referencedSegmentIds,
             Map<String, RecordId> strings, Map<Template, RecordId> templates)
{
-        tar.put(segmentId.toString().getBytes(UTF_8));
-        tar.putInt(referencedSegmentIds.size());
+        int size = 8 + 4 + 4 + 16 + 16 * referencedSegmentIds.size() + length;
+
+        if (0x200 + ((size + 0x1ff) & ~0x1ff) + 2 * 0x200 > rw.remaining()) {
+            rw.force();
+            String name = String.format(FILE_NAME_FORMAT, index);
+            File file = new File(directory, name);
+            try {
+                RandomAccessFile f = new RandomAccessFile(file, "rw");
+                try {
+                    rw = f.getChannel().map(READ_WRITE, 0, FILE_SIZE);
+                    ro = rw.asReadOnlyBuffer();
+                } finally {
+                    f.close();
+                }
+            } catch (IOException e) {
+                throw new RuntimeException("Unable to create a new segment", e);
+            }
+        }
+
+        rw.put(createTarHeader(segmentId.toString(), size));
+
+        rw.putLong(MAGIC_BYTES);
+        rw.putInt(length);
+        rw.putInt(referencedSegmentIds.size());
+        rw.putLong(segmentId.getMostSignificantBits());
+        rw.putLong(segmentId.getLeastSignificantBits());
         for (UUID referencedSegmentId : referencedSegmentIds) {
-            tar.putLong(referencedSegmentId.getMostSignificantBits());
-            tar.putLong(referencedSegmentId.getLeastSignificantBits());
+            rw.putLong(referencedSegmentId.getMostSignificantBits());
+            rw.putLong(referencedSegmentId.getLeastSignificantBits());
         }
-        tar.putInt(length);
-        int position = tar.position();
-        tar.put(data, offset, length);
-
-        ByteBuffer buffer = tar.asReadOnlyBuffer();
-        buffer.position(position);
-        buffer.limit(position + length);
-        buffer = buffer.slice();
+
+        ro.position(rw.position());
+        rw.put(data, offset, length);
+        ro.limit(rw.position());
+        ByteBuffer buffer = ro.slice();
+        ro.limit(rw.limit());
+
+        rw.position((rw.position() + 0x1ff) & ~0x1ff);
+        ro.position(rw.position());
 
         Segment segment = new Segment(
                 this, segmentId, buffer,
                 referencedSegmentIds, strings, templates);
-        if (segments.putIfAbsent(segmentId, segment) != null) {
-            throw new IllegalStateException(
-                    "Segment override: " + segmentId);
-        }
+        checkState(segments.put(segmentId, segment) == null);
     }
 
     @Override
@@ -130,4 +219,76 @@ public class FileStore implements Segmen
         }
     }
 
+    private static byte[] createTarHeader(String name, int length) {
+        byte[] header = new byte[] {
+                // File name
+                0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+                0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+                0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+                0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+                0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+                0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+                0, 0, 0, 0,
+                // File mode
+                '0', '0', '0', '0', '4', '0', '0', 0,
+                // User's numeric user ID
+                '0', '0', '0', '0', '0', '0', '0', 0,
+                // Group's numeric user ID
+                '0', '0', '0', '0', '0', '0', '0', 0,
+                // File size in bytes (octal basis)
+                '0', '0', '0', '0', '0', '0', '0', '0', '0', '0', '0',  0,
+                // Last modification time in numeric Unix time format (octal)
+                '0', '0', '0', '0', '0', '0', '0', '0', '0', '0', '0',  0,
+                // Checksum for header record
+                ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ',
+                // Type flag
+                '0',
+                // Name of linked file
+                0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+                0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+                0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+                0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+                0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+                0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+                0, 0, 0, 0,
+                // unused
+                0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+                0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+                0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+                0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+                0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+                0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+                0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+                0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+                0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+                0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+                0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+                0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+                0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+                0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+                0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+                0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+        };
+
+        byte[] n = name.getBytes(UTF_8);
+        System.arraycopy(n, 0, header, 0, n.length);
+
+        byte[] l = Integer.toOctalString(length).getBytes(UTF_8);
+        System.arraycopy(l, 0, header, 124 + 11 - l.length, l.length);
+
+        long time = System.currentTimeMillis() / 1000;
+        byte[] t = Long.toOctalString(time).getBytes(UTF_8);
+        System.arraycopy(t, 0, header, 136 + 11 - t.length, t.length);
+
+        int checksum = 0;
+        for (int i = 0; i < header.length; i++) {
+            checksum += header[i] & 0xff;
+        }
+        byte[] c = Integer.toOctalString(checksum).getBytes(UTF_8);
+        System.arraycopy(c, 0, header, 148 + 6 - c.length, c.length);
+        header[148 + 6] = 0;
+
+        return header;
+    }
+
 }

Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/memory/MemoryStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/memory/MemoryStore.java?rev=1478277&r1=1478276&r2=1478277&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/memory/MemoryStore.java
(original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/memory/MemoryStore.java
Thu May  2 06:04:08 2013
@@ -49,6 +49,10 @@ public class MemoryStore implements Segm
     }
 
     @Override
+    public void close() {
+    }
+
+    @Override
     public synchronized Journal getJournal(final String name) {
         Journal journal = journals.get(name);
         if (journal == null) {

Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/mongo/MongoStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/mongo/MongoStore.java?rev=1478277&r1=1478276&r2=1478277&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/mongo/MongoStore.java
(original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/mongo/MongoStore.java
Thu May  2 06:04:08 2013
@@ -76,6 +76,10 @@ public class MongoStore implements Segme
     }
 
     @Override
+    public void close() {
+    }
+
+    @Override
     public synchronized Journal getJournal(String name) {
         Journal journal = journals.get(name);
         if (journal == null) {



Mime
View raw message