jackrabbit-oak-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ju...@apache.org
Subject svn commit: r1560780 - in /jackrabbit/oak/trunk/oak-core/src: main/java/org/apache/jackrabbit/oak/plugins/segment/ main/java/org/apache/jackrabbit/oak/plugins/segment/file/ test/java/org/apache/jackrabbit/oak/plugins/segment/
Date Thu, 23 Jan 2014 18:25:41 GMT
Author: jukka
Date: Thu Jan 23 18:25:41 2014
New Revision: 1560780

URL: http://svn.apache.org/r1560780
Log:
OAK-1333: SegmentMK: Support for Blobs in external storage

Patch by Dominique Pfister

Added:
    jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/ExternalBlob.java
  (with props)
    jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileBlob.java
      - copied, changed from r1560720, jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentBlob.java
    jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/ExternalBlobTest.java
  (with props)
Modified:
    jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/AbstractStore.java
    jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Segment.java
    jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentBlob.java
    jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentPropertyState.java
    jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentStore.java
    jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentWriter.java
    jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java

Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/AbstractStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/AbstractStore.java?rev=1560780&r1=1560779&r2=1560780&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/AbstractStore.java
(original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/AbstractStore.java
Thu Jan 23 18:25:41 2014
@@ -24,6 +24,7 @@ import java.util.Set;
 import java.util.UUID;
 
 import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
 
 import org.apache.jackrabbit.oak.cache.CacheLIRS;
 
@@ -172,4 +173,9 @@ public abstract class AbstractStore impl
         return type.isInstance(object) && ((Record) object).getStore() == this;
     }
 
+    @Nullable
+    @Override
+    public ExternalBlob readBlob(String reference) {
+        return null;
+    }
 }

Added: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/ExternalBlob.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/ExternalBlob.java?rev=1560780&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/ExternalBlob.java
(added)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/ExternalBlob.java
Thu Jan 23 18:25:41 2014
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.jackrabbit.oak.plugins.segment;
+
+import org.apache.jackrabbit.oak.api.Blob;
+
+/**
+ * Marks a blob that is external.
+ */
+public interface ExternalBlob extends Blob {
+
+    /**
+     * Return a reference to this external blob.
+     *
+     * @return reference
+     */
+    public String getReference();
+}

Propchange: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/ExternalBlob.java
------------------------------------------------------------------------------
    svn:eol-style = native

Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Segment.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Segment.java?rev=1560780&r1=1560779&r2=1560780&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Segment.java
(original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Segment.java
Thu Jan 23 18:25:41 2014
@@ -440,6 +440,12 @@ public class Segment {
         }
     }
 
+    SegmentBlob createBlob(int offset) {
+        RecordId id = new RecordId(uuid, offset);
+        int n = readByte(offset) & 0xff;
+        return new SegmentBlob(this, id, (n & 0xe0) == 0xe0);
+    }
+
     SegmentStream readStream(int offset) {
         RecordId id = new RecordId(uuid, offset);
         int pos = pos(offset, 1);
@@ -462,6 +468,25 @@ public class Segment {
         }
     }
 
+    String readBlobReference(int offset) {
+        int pos = pos(offset, 1);
+
+        int length = (data.get(pos++) & 0x1f) << 8
+                | (data.get(pos++) & 0xff);
+
+        byte[] bytes = new byte[length];
+        ByteBuffer buffer = data.duplicate();
+        buffer.position(pos + 8); // skip blob length
+        buffer.get(bytes);
+        return new String(bytes, Charsets.UTF_8);
+    }
+
+    long readBlobLength(int offset) {
+        long high = readInt(offset + 2);
+        long low = readInt(offset + 6);
+        return high << 32 | low;
+    }
+
     //------------------------------------------------------------< Object >--
 
     @Override

Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentBlob.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentBlob.java?rev=1560780&r1=1560779&r2=1560780&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentBlob.java
(original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentBlob.java
Thu Jan 23 18:25:41 2014
@@ -21,20 +21,34 @@ import javax.annotation.Nonnull;
 import org.apache.jackrabbit.oak.api.Blob;
 import org.apache.jackrabbit.oak.plugins.memory.AbstractBlob;
 
+import java.io.InputStream;
+
 class SegmentBlob extends Record implements Blob {
 
-    SegmentBlob(Segment segment, RecordId id) {
+    private boolean external;
+
+    SegmentBlob(Segment segment, RecordId id, boolean external) {
         super(segment, id);
+
+        this.external = external;
     }
 
     @Override @Nonnull
-    public SegmentStream getNewStream() {
+    public InputStream getNewStream() {
+        if (external) {
+            String refererence = getSegment().readBlobReference(getOffset());
+            return getStore().readBlob(refererence).getNewStream();
+        }
         return getSegment().readStream(getOffset());
     }
 
     @Override
     public long length() {
-        SegmentStream stream = getNewStream();
+        if (external) {
+            return getSegment().readBlobLength(getOffset());
+        }
+
+        SegmentStream stream = (SegmentStream) getNewStream();
         try {
             return stream.getLength();
         } finally {

Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentPropertyState.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentPropertyState.java?rev=1560780&r1=1560779&r2=1560780&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentPropertyState.java
(original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentPropertyState.java
Thu Jan 23 18:25:41 2014
@@ -162,7 +162,7 @@ class SegmentPropertyState extends Recor
     @SuppressWarnings("unchecked")
     private <T> T getValue(Segment segment, RecordId id, Type<T> type) {
         if (type == BINARY) {
-            return (T) new SegmentBlob(segment, id); // load binaries lazily
+            return (T) segment.createBlob(id.getOffset()); // load binaries lazily
         }
 
         String value = segment.readString(id);

Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentStore.java?rev=1560780&r1=1560779&r2=1560780&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentStore.java
(original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentStore.java
Thu Jan 23 18:25:41 2014
@@ -16,6 +16,7 @@
  */
 package org.apache.jackrabbit.oak.plugins.segment;
 
+import javax.annotation.Nullable;
 import java.util.UUID;
 
 public interface SegmentStore {
@@ -51,4 +52,11 @@ public interface SegmentStore {
      */
     boolean isInstance(Object object, Class<? extends Record> type);
 
+    /**
+     * Read a blob from external storage.
+     *
+     * @param reference blob reference
+     * @return external blob
+     */
+    ExternalBlob readBlob(String reference);
 }

Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentWriter.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentWriter.java?rev=1560780&r1=1560779&r2=1560780&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentWriter.java
(original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentWriter.java
Thu Jan 23 18:25:41 2014
@@ -487,6 +487,31 @@ public class SegmentWriter {
     }
 
     /**
+     * Write a reference to an external blob.
+     *
+     * @param reference reference
+     * @param blobLength blob length
+     * @return record id
+     */
+    private synchronized RecordId writeValueRecord(String reference, long blobLength) {
+        byte[] data = reference.getBytes(Charsets.UTF_8);
+        int length = data.length;
+
+        checkArgument(length < 8192);
+
+        RecordId id = prepare(RecordType.VALUE, 2 + 8 + length);
+        int len = length | 0xE000;
+        buffer[position++] = (byte) (len >> 8);
+        buffer[position++] = (byte) len;
+
+        writeLong(blobLength);
+
+        System.arraycopy(data, 0, buffer, position, length);
+        position += length;
+        return id;
+    }
+
+    /**
      * Writes a block record containing the given block of bytes.
      *
      * @param bytes source buffer
@@ -643,13 +668,20 @@ public class SegmentWriter {
     }
 
     public SegmentBlob writeBlob(Blob blob) throws IOException {
-        if (store.isInstance(blob, SegmentBlob.class)) {
+        if (blob instanceof ExternalBlob) {
+            return writeBlob((ExternalBlob) blob);
+        } else if (store.isInstance(blob, SegmentBlob.class)) {
             return (SegmentBlob) blob;
         } else {
             return writeStream(blob.getNewStream());
         }
     }
 
+    private SegmentBlob writeBlob(ExternalBlob blob) {
+        RecordId id = writeValueRecord(blob.getReference(), blob.length());
+        return new SegmentBlob(dummySegment, id, true);
+    }
+
     /**
      * Writes a stream value record. The given stream is consumed
      * <em>and closed</em> by this method.
@@ -669,7 +701,7 @@ public class SegmentWriter {
                 Closeables.close(stream, threw);
             }
         }
-        return new SegmentBlob(dummySegment, id);
+        return new SegmentBlob(dummySegment, id, false);
     }
 
     private RecordId internalWriteStream(InputStream stream)

Copied: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileBlob.java
(from r1560720, jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentBlob.java)
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileBlob.java?p2=jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileBlob.java&p1=jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentBlob.java&r1=1560720&r2=1560780&rev=1560780&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentBlob.java
(original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileBlob.java
Thu Jan 23 18:25:41 2014
@@ -14,49 +14,58 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.jackrabbit.oak.plugins.segment;
+package org.apache.jackrabbit.oak.plugins.segment.file;
+
+import org.apache.jackrabbit.oak.plugins.segment.ExternalBlob;
 
 import javax.annotation.Nonnull;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
 
-import org.apache.jackrabbit.oak.api.Blob;
-import org.apache.jackrabbit.oak.plugins.memory.AbstractBlob;
+public class FileBlob implements ExternalBlob {
 
-class SegmentBlob extends Record implements Blob {
+    private final String path;
 
-    SegmentBlob(Segment segment, RecordId id) {
-        super(segment, id);
+    public FileBlob(String path) {
+        this.path = path;
     }
 
-    @Override @Nonnull
-    public SegmentStream getNewStream() {
-        return getSegment().readStream(getOffset());
+    public String getReference() {
+        return path;
     }
 
+    @Nonnull
     @Override
-    public long length() {
-        SegmentStream stream = getNewStream();
+    public InputStream getNewStream() {
         try {
-            return stream.getLength();
-        } finally {
-            stream.close();
+            return new FileInputStream(getFile());
+        } catch (IOException e) {
+            throw new RuntimeException(e);
         }
     }
 
-    //------------------------------------------------------------< Object >--
+    @Override
+    public long length() {
+        return getFile().length();
+    }
+
+    private File getFile() {
+        return new File(path);
+    }
 
     @Override
-    public boolean equals(Object object) {
-        if (object == this || fastEquals(this, object)) {
-            return true;
-        } else {
-            return object instanceof Blob
-                    && AbstractBlob.equal(this, (Blob) object);
+    public boolean equals(Object obj) {
+        if (obj instanceof FileBlob) {
+            FileBlob other = (FileBlob) obj;
+            return this.path.equals(other.path);
         }
+        return super.equals(obj);
     }
 
     @Override
     public int hashCode() {
-        return 0;
+        return path.hashCode();
     }
-
 }

Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java?rev=1560780&r1=1560779&r2=1560780&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java
(original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java
Thu Jan 23 18:25:41 2014
@@ -38,6 +38,7 @@ import java.util.concurrent.atomic.Atomi
 import javax.annotation.Nonnull;
 
 import org.apache.jackrabbit.oak.plugins.segment.AbstractStore;
+import org.apache.jackrabbit.oak.plugins.segment.ExternalBlob;
 import org.apache.jackrabbit.oak.plugins.segment.Journal;
 import org.apache.jackrabbit.oak.plugins.segment.RecordId;
 import org.apache.jackrabbit.oak.plugins.segment.Segment;
@@ -334,4 +335,9 @@ public class FileStore extends AbstractS
         super.deleteSegment(segmentId);
     }
 
+    @Override
+    public ExternalBlob readBlob(String reference) {
+        return new FileBlob(reference);
+    }
+
 }

Added: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/ExternalBlobTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/ExternalBlobTest.java?rev=1560780&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/ExternalBlobTest.java
(added)
+++ jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/ExternalBlobTest.java
Thu Jan 23 18:25:41 2014
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.oak.plugins.segment;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.jackrabbit.oak.api.Blob;
+import org.apache.jackrabbit.oak.plugins.memory.AbstractBlob;
+import org.apache.jackrabbit.oak.plugins.segment.file.FileBlob;
+import org.apache.jackrabbit.oak.api.Type;
+import org.apache.jackrabbit.oak.plugins.segment.SegmentNodeStore;
+import org.apache.jackrabbit.oak.plugins.segment.SegmentStore;
+import org.apache.jackrabbit.oak.plugins.segment.file.FileStore;
+import org.apache.jackrabbit.oak.spi.commit.EmptyHook;
+import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
+import org.apache.jackrabbit.oak.spi.state.NodeState;
+import org.junit.After;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Random;
+
+import static junit.framework.Assert.assertTrue;
+
+public class ExternalBlobTest {
+
+    private SegmentStore store;
+    private SegmentNodeStore nodeStore;
+    private FileBlob fileBlob;
+
+    @Test
+    public void testCreateAndRead() throws Exception {
+        SegmentNodeStore nodeStore = getNodeStore();
+
+        NodeState state = nodeStore.getRoot().getChildNode("hello");
+        if (!state.exists()) {
+            NodeBuilder builder = nodeStore.getRoot().builder();
+            builder.child("hello");
+            nodeStore.merge(builder, EmptyHook.INSTANCE, null);
+        }
+
+        Blob blob = getFileBlob();
+        NodeBuilder builder = nodeStore.getRoot().builder();
+        builder.getChildNode("hello").setProperty("world", blob);
+        nodeStore.merge(builder, EmptyHook.INSTANCE, null);
+
+        state = nodeStore.getRoot().getChildNode("hello");
+        blob = state.getProperty("world").getValue(Type.BINARY);
+
+        assertTrue("Blob written and read must be equal",
+                AbstractBlob.equal(blob, getFileBlob()));
+    }
+
+    @After
+    public void close() {
+        if (store != null) {
+            store.close();
+        }
+    }
+
+    protected SegmentNodeStore getNodeStore() throws IOException {
+        if (nodeStore == null) {
+            store = new FileStore(new File("target", "ExternalBlobTest"), 256, false);
+            nodeStore = new SegmentNodeStore(store);
+        }
+        return nodeStore;
+    }
+
+    private FileBlob getFileBlob() throws IOException {
+        if (fileBlob == null) {
+            File file = File.createTempFile("blob", "tmp");
+            file.deleteOnExit();
+
+            byte[] data = new byte[2345];
+            new Random().nextBytes(data);
+            FileUtils.writeByteArrayToFile(file, data);
+
+            fileBlob = new FileBlob(file.getPath());
+        }
+        return fileBlob;
+    }
+}

Propchange: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/ExternalBlobTest.java
------------------------------------------------------------------------------
    svn:eol-style = native



Mime
View raw message