jackrabbit-oak-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ju...@apache.org
Subject svn commit: r1502238 - in /jackrabbit/oak/trunk: oak-core/src/main/java/org/apache/jackrabbit/oak/core/ oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/memory/ oak-core/src/main/java/org/apache/jackrabbit/oak/spi/state/ oak-lucene/src/main/jav...
Date Thu, 11 Jul 2013 14:43:59 GMT
Author: jukka
Date: Thu Jul 11 14:43:59 2013
New Revision: 1502238

URL: http://svn.apache.org/r1502238
Log:
OAK-895: Random access for Lucene index binaries

Split the Lucene index files to arrays of smaller blobs so we can read and modify them in
a more granular manner.

Modified:
    jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/core/SecureNodeBuilder.java
    jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/memory/MemoryNodeBuilder.java
    jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/spi/state/NodeBuilder.java
    jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/spi/state/ReadOnlyBuilder.java
    jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorContext.java
    jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/OakDirectory.java

Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/core/SecureNodeBuilder.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/core/SecureNodeBuilder.java?rev=1502238&r1=1502237&r2=1502238&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/core/SecureNodeBuilder.java
(original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/core/SecureNodeBuilder.java
Thu Jul 11 14:43:59 2013
@@ -24,12 +24,16 @@ import static org.apache.jackrabbit.oak.
 import static org.apache.jackrabbit.oak.api.Type.NAME;
 import static org.apache.jackrabbit.oak.api.Type.NAMES;
 
+import java.io.IOException;
+import java.io.InputStream;
+
 import javax.annotation.CheckForNull;
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 
 import com.google.common.base.Predicate;
 
+import org.apache.jackrabbit.oak.api.Blob;
 import org.apache.jackrabbit.oak.api.PropertyState;
 import org.apache.jackrabbit.oak.api.Type;
 import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
@@ -246,6 +250,11 @@ class SecureNodeBuilder implements NodeB
         }
     }
 
+    @Override
+    public Blob createBlob(InputStream stream) throws IOException {
+        return builder.createBlob(stream);
+    }
+
     //------------------------------------------------------< inner classes >---
 
     /**

Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/memory/MemoryNodeBuilder.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/memory/MemoryNodeBuilder.java?rev=1502238&r1=1502237&r2=1502238&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/memory/MemoryNodeBuilder.java
(original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/memory/MemoryNodeBuilder.java
Thu Jul 11 14:43:59 2013
@@ -25,13 +25,19 @@ import static org.apache.jackrabbit.oak.
 import static org.apache.jackrabbit.oak.api.Type.NAMES;
 import static org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState.EMPTY_NODE;
 
+import java.io.IOException;
+import java.io.InputStream;
+
 import javax.annotation.Nonnull;
 
+import org.apache.jackrabbit.oak.api.Blob;
 import org.apache.jackrabbit.oak.api.PropertyState;
 import org.apache.jackrabbit.oak.api.Type;
 import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
 import org.apache.jackrabbit.oak.spi.state.NodeState;
 
+import com.google.common.io.ByteStreams;
+
 /**
  * In-memory node state builder.
  * <p>
@@ -355,6 +361,15 @@ public class MemoryNodeBuilder implement
         return this;
     }
 
+    @Override
+    public Blob createBlob(InputStream stream) throws IOException {
+        try {
+            return new ArrayBasedBlob(ByteStreams.toByteArray(stream));
+        } finally {
+            stream.close();
+        }
+    }
+
     /**
      * @return path of this builder. For debugging purposes only
      */

Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/spi/state/NodeBuilder.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/spi/state/NodeBuilder.java?rev=1502238&r1=1502237&r2=1502238&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/spi/state/NodeBuilder.java
(original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/spi/state/NodeBuilder.java
Thu Jul 11 14:43:59 2013
@@ -16,9 +16,13 @@
  */
 package org.apache.jackrabbit.oak.spi.state;
 
+import java.io.IOException;
+import java.io.InputStream;
+
 import javax.annotation.CheckForNull;
 import javax.annotation.Nonnull;
 
+import org.apache.jackrabbit.oak.api.Blob;
 import org.apache.jackrabbit.oak.api.PropertyState;
 import org.apache.jackrabbit.oak.api.Type;
 
@@ -339,4 +343,6 @@ public interface NodeBuilder {
     @Nonnull
     NodeBuilder removeProperty(String name);
 
+    Blob createBlob(InputStream stream) throws IOException;
+
 }

Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/spi/state/ReadOnlyBuilder.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/spi/state/ReadOnlyBuilder.java?rev=1502238&r1=1502237&r2=1502238&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/spi/state/ReadOnlyBuilder.java
(original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/spi/state/ReadOnlyBuilder.java
Thu Jul 11 14:43:59 2013
@@ -16,9 +16,13 @@
  */
 package org.apache.jackrabbit.oak.spi.state;
 
+import java.io.IOException;
+import java.io.InputStream;
+
 import javax.annotation.CheckForNull;
 import javax.annotation.Nonnull;
 
+import org.apache.jackrabbit.oak.api.Blob;
 import org.apache.jackrabbit.oak.api.PropertyState;
 import org.apache.jackrabbit.oak.api.Type;
 
@@ -168,4 +172,9 @@ public class ReadOnlyBuilder implements 
         throw unsupported();
     }
 
+    @Override
+    public Blob createBlob(InputStream stream) throws IOException {
+        throw unsupported();
+    }
+
 }

Modified: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorContext.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorContext.java?rev=1502238&r1=1502237&r2=1502238&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorContext.java
(original)
+++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorContext.java
Thu Jul 11 14:43:59 2013
@@ -66,8 +66,7 @@ public class LuceneIndexEditorContext {
             throws IOException {
         String path = getString(definition, PERSISTENCE_PATH);
         if (path == null) {
-            return new OakDirectory(
-                    definition.child(INDEX_DATA_CHILD_NAME));
+            return new OakDirectory(definition.child(INDEX_DATA_CHILD_NAME));
         } else {
             // try {
             File file = new File(path);

Modified: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/OakDirectory.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/OakDirectory.java?rev=1502238&r1=1502237&r2=1502238&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/OakDirectory.java
(original)
+++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/OakDirectory.java
Thu Jul 11 14:43:59 2013
@@ -16,13 +16,26 @@
  */
 package org.apache.jackrabbit.oak.plugins.index.lucene;
 
-import static org.apache.jackrabbit.oak.api.Type.BINARY;
+import static com.google.common.base.Preconditions.checkElementIndex;
+import static com.google.common.base.Preconditions.checkNotNull;
+import static com.google.common.base.Preconditions.checkPositionIndexes;
+import static com.google.common.base.Preconditions.checkState;
+import static com.google.common.collect.Lists.newArrayList;
+import static org.apache.jackrabbit.JcrConstants.JCR_DATA;
+import static org.apache.jackrabbit.JcrConstants.JCR_LASTMODIFIED;
+import static org.apache.jackrabbit.oak.api.Type.BINARIES;
 
+import java.io.ByteArrayInputStream;
+import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.InputStream;
 import java.util.Collection;
+import java.util.List;
 
 import com.google.common.collect.Iterables;
+import com.google.common.io.ByteStreams;
+
+import org.apache.jackrabbit.oak.api.Blob;
 import org.apache.jackrabbit.oak.api.PropertyState;
 import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
 import org.apache.lucene.store.Directory;
@@ -62,30 +75,31 @@ class OakDirectory extends Directory {
 
     @Override
     public long fileLength(String name) throws IOException {
-        if (!fileExists(name)) {
-            return 0;
-        }
-
-        NodeBuilder fileBuilder = directoryBuilder.child(name);
-        PropertyState property = fileBuilder.getProperty("jcr:data");
-        if (property == null || property.isArray()) {
-            return 0;
+        NodeBuilder file = directoryBuilder.getChildNode(name);
+        OakIndexInput input = new OakIndexInput(name, file);
+        try {
+            return input.length();
+        } finally {
+            input.close();
         }
-
-        return property.size();
     }
 
     @Override
     public IndexOutput createOutput(String name, IOContext context)
             throws IOException {
-        return new OakIndexOutput(name);
+        return new OakIndexOutput(name, directoryBuilder.child(name));
     }
 
 
     @Override
     public IndexInput openInput(String name, IOContext context)
             throws IOException {
-        return new OakIndexInput(name);
+        NodeBuilder file = directoryBuilder.getChildNode(name);
+        if (file.exists()) {
+            return new OakIndexInput(name, file);
+        } else {
+            throw new FileNotFoundException(name);
+        }
     }
 
     @Override
@@ -98,88 +112,212 @@ class OakDirectory extends Directory {
         // do nothing
     }
 
-    protected byte[] readFile(String name) throws IOException {
-        if (!fileExists(name)) {
-            return new byte[0];
+    private static final int BLOB_SIZE = 4092;
+
+    private static class OakIndexFile {
+
+        private final String name;
+
+        private final NodeBuilder file;
+
+        private long position = 0;
+
+        private long length;
+
+        private final List<Blob> data;
+
+        private boolean dataModified = false;
+
+        private int index = -1;
+
+        private final byte[] blob = new byte[BLOB_SIZE];
+
+        private boolean blobModified = false;
+
+        public OakIndexFile(String name, NodeBuilder file) {
+            this.name = name;
+            this.file = file;
+
+            PropertyState property = file.getProperty(JCR_DATA);
+            if (property != null && property.getType() == BINARIES) {
+                this.data = newArrayList(property.getValue(BINARIES));
+            } else {
+                this.data = newArrayList();
+            }
+
+            this.length = data.size() * BLOB_SIZE;
+            if (!data.isEmpty()) {
+                Blob last = data.get(data.size() - 1);
+                this.length -= BLOB_SIZE - last.length();
+            }
         }
 
-        NodeBuilder fileBuilder = directoryBuilder.child(name);
-        PropertyState property = fileBuilder.getProperty("jcr:data");
-        if (property == null || property.isArray()) {
-            return new byte[0];
+        private OakIndexFile(OakIndexFile that) {
+            this.name = that.name;
+            this.file = that.file;
+
+            this.position = that.position;
+            this.length = that.length;
+            this.data = newArrayList(that.data);
+            this.dataModified = that.dataModified;
+        }
+
+        private void loadBlob(int i) throws IOException {
+            checkElementIndex(i, data.size());
+            if (index != i) {
+                flushBlob();
+                checkState(!blobModified);
+
+                int n = (int) Math.min(BLOB_SIZE, length - i * BLOB_SIZE);
+                InputStream stream = data.get(i).getNewStream();
+                try {
+                    ByteStreams.readFully(stream, blob, 0, n);
+                } finally {
+                    stream.close();
+                }
+                index = i;
+            }
         }
 
-        InputStream stream = property.getValue(BINARY).getNewStream();
-        try {
-            byte[] buffer = new byte[(int) property.size()];
+        private void flushBlob() throws IOException {
+            if (blobModified) {
+                System.out.format("%s flush blob %d%n", name, index);
+                int n = (int) Math.min(BLOB_SIZE, length - index * BLOB_SIZE);
+                Blob b = file.createBlob(new ByteArrayInputStream(blob, 0, n));
+                if (index < data.size()) {
+                    data.set(index, b);
+                } else {
+                    checkState(index == data.size());
+                    data.add(b);
+                }
+                dataModified = true;
+                blobModified = false;
+            }
+        }
+
+        public void seek(long pos) throws IOException {
+            // seek() may be called with pos == length
+            // see https://issues.apache.org/jira/browse/LUCENE-1196
+            if (pos < 0 || pos > length) {
+                throw new IOException("Invalid seek request");
+            } else {
+                position = pos;
+            }
+        }
+
+        public void readBytes(byte[] b, int offset, int len)
+                throws IOException {
+            checkPositionIndexes(offset, offset + len, checkNotNull(b).length);
+
+            if (len < 0 || position + len > length) {
+                throw new IOException("Invalid byte range request");
+            }
+
+            int i = (int) (position / BLOB_SIZE);
+            int o = (int) (position % BLOB_SIZE);
+            while (len > 0) {
+                loadBlob(i);
+
+                int l = Math.min(len, BLOB_SIZE - o);
+                System.arraycopy(blob, o, b, offset, l);
+
+                offset += l;
+                len -= l;
+                position += l;
+
+                i++;
+                o = 0;
+            }
+        }
 
-            int size = 0;
-            do {
-                int n = stream.read(buffer, size, buffer.length - size);
-                if (n == -1) {
-                    throw new IOException(
-                            "Unexpected end of index file: " + name);
+        public void writeBytes(byte[] b, int offset, int len)
+                throws IOException {
+            int i = (int) (position / BLOB_SIZE);
+            int o = (int) (position % BLOB_SIZE);
+            while (len > 0) {
+                int l = Math.min(len, BLOB_SIZE - o);
+
+                if (index != i) {
+                    if (o > 0 || (l < BLOB_SIZE && position + l < length))
{
+                        loadBlob(i);
+                    } else {
+                        flushBlob();
+                        index = i;
+                    }
                 }
-                size += n;
-            } while (size < buffer.length);
+                System.arraycopy(b, offset, blob, o, l);
+                blobModified = true;
 
-            return buffer;
-        } finally {
-            stream.close();
+                offset += l;
+                len -= l;
+                position += l;
+                length = Math.max(length, position);
+
+                i++;
+                o = 0;
+            }
         }
-    }
 
-    private final class OakIndexInput extends IndexInput {
+        public void flush() throws IOException {
+            flushBlob();
+            if (dataModified) {
+                file.setProperty(JCR_LASTMODIFIED, System.currentTimeMillis());
+                file.setProperty(JCR_DATA, data, BINARIES);
+                dataModified = false;
+            }
+        }
 
-        private final byte[] data;
+        @Override
+        public String toString() {
+            return name;
+        }
 
-        private int position;
+    }
+
+    private static class OakIndexInput extends IndexInput {
+
+        private final OakIndexFile file;
 
-        public OakIndexInput(String name) throws IOException {
+        public OakIndexInput(String name, NodeBuilder file) {
             super(name);
-            this.data = readFile(name);
-            this.position = 0;
+            this.file = new OakIndexFile(name, file);
+        }
+
+        private OakIndexInput(OakIndexInput that) {
+            super(that.toString());
+            this.file = new OakIndexFile(that.file);
         }
 
         @Override
-        public void readBytes(byte[] b, int offset, int len)
-                throws IOException {
-            if (len < 0 || position + len > data.length) {
-                throw new IOException("Invalid byte range request");
-            } else {
-                System.arraycopy(data, position, b, offset, len);
-                position += len;
-            }
+        public OakIndexInput clone() {
+            return new OakIndexInput(this);
+        }
+
+        @Override
+        public void readBytes(byte[] b, int o, int n) throws IOException {
+            file.readBytes(b, o, n);
         }
 
         @Override
         public byte readByte() throws IOException {
-            if (position >= data.length) {
-                throw new IOException("Invalid byte range request");
-            } else {
-                return data[position++];
-            }
+            byte[] b = new byte[1];
+            readBytes(b, 0, 1);
+            return b[0];
         }
 
         @Override
         public void seek(long pos) throws IOException {
-            //seek() may be called with pos == data.length
-            //see https://issues.apache.org/jira/browse/LUCENE-1196
-            if (pos < 0 || pos > data.length) {
-                throw new IOException("Invalid seek request");
-            } else {
-                position = (int) pos;
-            }
+            file.seek(pos);
         }
 
         @Override
         public long length() {
-            return data.length;
+            return file.length;
         }
 
         @Override
         public long getFilePointer() {
-            return position;
+            return file.position;
         }
 
         @Override
@@ -191,77 +329,48 @@ class OakDirectory extends Directory {
 
     private final class OakIndexOutput extends IndexOutput {
 
-        private final String name;
-
-        private byte[] buffer;
-
-        private int size;
+        private final OakIndexFile file;
 
-        private int position;
-
-        public OakIndexOutput(String name) throws IOException {
-            this.name = name;
-            this.buffer = readFile(name);
-            this.size = buffer.length;
-            this.position = 0;
+        public OakIndexOutput(String name, NodeBuilder file) throws IOException {
+            this.file = new OakIndexFile(name, file);
         }
 
         @Override
         public long length() {
-            return size;
+            return file.length;
         }
 
         @Override
         public long getFilePointer() {
-            return position;
+            return file.position;
         }
 
         @Override
         public void seek(long pos) throws IOException {
-            if (pos < 0 || pos > Integer.MAX_VALUE) {
-                throw new IOException("Invalid file position: " + pos);
-            }
-            this.position = (int) pos;
+            file.seek(pos);
         }
 
         @Override
-        public void writeBytes(byte[] b, int offset, int length) {
-            while (position + length > buffer.length) {
-                byte[] tmp = new byte[Math.max(4096, buffer.length * 2)];
-                System.arraycopy(buffer, 0, tmp, 0, size);
-                buffer = tmp;
-            }
-
-            System.arraycopy(b, offset, buffer, position, length);
-
-            position += length;
-            if (position > size) {
-                size = position;
-            }
+        public void writeBytes(byte[] b, int offset, int length)
+                throws IOException {
+            file.writeBytes(b, offset, length);
         }
 
         @Override
-        public void writeByte(byte b) {
+        public void writeByte(byte b) throws IOException {
             writeBytes(new byte[] { b }, 0, 1);
         }
 
         @Override
         public void flush() throws IOException {
-            byte[] data = buffer;
-            if (data.length > size) {
-                data = new byte[size];
-                System.arraycopy(buffer, 0, data, 0, size);
-            }
-
-            NodeBuilder fileBuilder = directoryBuilder.child(name);
-            fileBuilder.setProperty("jcr:lastModified", System.currentTimeMillis());
-            fileBuilder.setProperty("jcr:data", data);
+            file.flush();
         }
 
         @Override
         public void close() throws IOException {
             flush();
         }
+
     }
 
 }



Mime
View raw message