jackrabbit-oak-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ju...@apache.org
Subject svn commit: r1446326 - in /jackrabbit/oak/trunk/oak-core/src: main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentWriter.java test/java/org/apache/jackrabbit/oak/plugins/segment/RecordTest.java
Date Thu, 14 Feb 2013 21:05:56 GMT
Author: jukka
Date: Thu Feb 14 21:05:56 2013
New Revision: 1446326

URL: http://svn.apache.org/r1446326
Log:
OAK-593: Segment-based MK

Store large binaries entirely in separate bulk segments to make normal segments more useful
for caching.

Modified:
    jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentWriter.java
    jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/RecordTest.java

Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentWriter.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentWriter.java?rev=1446326&r1=1446325&r2=1446326&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentWriter.java
(original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentWriter.java
Thu Feb 14 21:05:56 2013
@@ -53,10 +53,6 @@ public class SegmentWriter {
 
     static final int BLOCK_SIZE = 1 << 12; // 4kB
 
-    static final int INLINE_BLOCKS = 16;
-
-    static final int INLINE_SIZE = INLINE_BLOCKS * BLOCK_SIZE; // 64kB
-
     private final SegmentStore store;
 
     private final int blocksPerSegment;
@@ -137,28 +133,6 @@ public class SegmentWriter {
         buffer.putInt(index << 24 | id.getOffset());
     }
 
-    private void writeInlineBlocks(
-            List<RecordId> blockIds, byte[] buffer, int offset, int length) {
-        int begin = offset;
-        int end = offset + length;
-        while (begin + BLOCK_SIZE <= end) {
-            blockIds.add(writeBlock(buffer, begin, BLOCK_SIZE));
-            begin += BLOCK_SIZE;
-        }
-        if (begin < end) {
-            blockIds.add(writeBlock(buffer, begin, end - begin));
-        }
-    }
-
-    private void writeBulkSegment(
-            List<RecordId> blockIds, byte[] buffer, int offset, int length) {
-        UUID segmentId = UUID.randomUUID();
-        store.createSegment(segmentId, buffer, offset, length);
-        for (int position = 0; position < length; position += BLOCK_SIZE) {
-            blockIds.add(new RecordId(segmentId, position));
-        }
-    }
-
     private synchronized RecordId writeListBucket(List<RecordId> bucket) {
         RecordId bucketId = prepare(0, bucket);
         for (RecordId id : bucket) {
@@ -355,49 +329,7 @@ public class SegmentWriter {
         RecordId id = SegmentStream.getRecordIdIfAvailable(stream);
         if (id == null) {
             try {
-                List<RecordId> blockIds = new ArrayList<RecordId>();
-
-                // First read the head of the stream. This covers most small
-                // values and the frequently accessed head of larger ones.
-                // The head gets inlined in the current segment.
-                byte[] head = new byte[INLINE_SIZE];
-                int headLength = ByteStreams.read(stream, head, 0, head.length);
-
-                if (headLength < 0x80) {
-                    id = prepare(1 + headLength);
-                    buffer.put((byte) headLength);
-                    buffer.put(head, 0, headLength);
-                } else if (headLength - 0x80 < 0x4000) {
-                    id = prepare(2 + headLength);
-                    buffer.putShort((short) ((headLength - 0x80) | 0x8000));
-                    buffer.put(head, 0, headLength);
-                } else {
-                    writeInlineBlocks(blockIds, head, 0, headLength);
-                    long length = headLength;
-
-                    // If the stream filled the full head buffer, it's likely
-                    // that the bulk of the data is still to come. Read it
-                    // in larger chunks and save in separate segments.
-                    if (headLength == head.length) {
-                        byte[] bulk = new byte[blockSegmentSize];
-                        int bulkLength = ByteStreams.read(
-                                stream, bulk, 0, bulk.length);
-                        while (bulkLength > INLINE_SIZE) {
-                            writeBulkSegment(blockIds, bulk, 0, bulkLength);
-                            length += bulkLength;
-                            bulkLength = ByteStreams.read(
-                                    stream, bulk, 0, bulk.length);
-                        }
-                        // The tail chunk of the stream is too small to put in
-                        // a separate segment, so we inline also it.
-                        if (bulkLength > 0) {
-                            writeInlineBlocks(blockIds, bulk, 0, bulkLength);
-                            length += bulkLength;
-                        }
-                    }
-
-                    id = writeValueRecord(length, writeList(blockIds));
-                }
+                id = internalWriteStream(stream);
             } finally {
                 stream.close();
             }
@@ -405,6 +337,50 @@ public class SegmentWriter {
         return id;
     }
 
+    private RecordId internalWriteStream(InputStream stream)
+            throws IOException {
+        // First read the head of the stream. This covers most small
+        // values and the frequently accessed head of larger ones.
+        // The head gets inlined in the current segment.
+        byte[] head = new byte[0x4080];
+        int headLength = ByteStreams.read(stream, head, 0, head.length);
+
+        if (headLength < 0x80) {
+            RecordId id = prepare(1 + headLength);
+            buffer.put((byte) headLength);
+            buffer.put(head, 0, headLength);
+            return id;
+        } else if (headLength < 0x4080) {
+            RecordId id = prepare(2 + headLength);
+            buffer.putShort((short) ((headLength - 0x80) | 0x8000));
+            buffer.put(head, 0, headLength);
+            return id;
+        } else {
+            // If the stream filled the full head buffer, it's likely
+            // that the bulk of the data is still to come. Read it
+            // in larger chunks and save in separate segments.
+
+            long length = 0;
+            List<RecordId> blockIds = new ArrayList<RecordId>();
+
+            byte[] bulk = new byte[blockSegmentSize];
+            System.arraycopy(head, 0, bulk, 0, headLength);
+            int bulkLength = headLength + ByteStreams.read(
+                    stream, bulk, headLength, bulk.length - headLength);
+            while (bulkLength > 0) {
+                UUID segmentId = UUID.randomUUID();
+                store.createSegment(segmentId, bulk, 0, bulkLength);
+                for (int pos = 0; pos < bulkLength; pos += BLOCK_SIZE) {
+                    blockIds.add(new RecordId(segmentId, pos));
+                }
+                length += bulkLength;
+                bulkLength = ByteStreams.read(stream, bulk, 0, bulk.length);
+            }
+
+            return writeValueRecord(length, writeList(blockIds));
+        }
+    }
+
     private RecordId writeProperty(PropertyState state) {
         Type<?> type = state.getType();
         int count = state.count();

Modified: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/RecordTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/RecordTest.java?rev=1446326&r1=1446325&r2=1446326&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/RecordTest.java
(original)
+++ jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/RecordTest.java
Thu Feb 14 21:05:56 2013
@@ -115,10 +115,12 @@ public class RecordTest {
     public void testStreamRecord() throws IOException {
         checkRandomStreamRecord(0);
         checkRandomStreamRecord(1);
+        checkRandomStreamRecord(0x79);
+        checkRandomStreamRecord(0x80);
+        checkRandomStreamRecord(0x4079);
+        checkRandomStreamRecord(0x4080);
         checkRandomStreamRecord(SegmentWriter.BLOCK_SIZE);
         checkRandomStreamRecord(SegmentWriter.BLOCK_SIZE + 1);
-        checkRandomStreamRecord(SegmentWriter.INLINE_SIZE);
-        checkRandomStreamRecord(SegmentWriter.INLINE_SIZE + 1);
         checkRandomStreamRecord(store.getMaxSegmentSize());
         checkRandomStreamRecord(store.getMaxSegmentSize() + 1);
         checkRandomStreamRecord(store.getMaxSegmentSize() * 2);



Mime
View raw message