jackrabbit-oak-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ju...@apache.org
Subject svn commit: r1547183 - in /jackrabbit/oak/trunk/oak-core/src: main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentWriter.java test/java/org/apache/jackrabbit/oak/plugins/segment/RecordTest.java
Date Mon, 02 Dec 2013 20:44:49 GMT
Author: jukka
Date: Mon Dec  2 20:44:49 2013
New Revision: 1547183

URL: http://svn.apache.org/r1547183
Log:
OAK-593: Segment-based MK

Write strings directly without roundtripping them trough ByteArrayInputStream

Modified:
    jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentWriter.java
    jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/RecordTest.java

Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentWriter.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentWriter.java?rev=1547183&r1=1547182&r2=1547183&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentWriter.java
(original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentWriter.java
Mon Dec  2 20:44:49 2013
@@ -36,7 +36,6 @@ import static org.apache.jackrabbit.oak.
 import static org.apache.jackrabbit.oak.plugins.segment.SegmentIdFactory.newBulkSegmentId;
 import static org.apache.jackrabbit.oak.plugins.segment.SegmentIdFactory.newDataSegmentId;
 
-import java.io.ByteArrayInputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.nio.ByteBuffer;
@@ -473,6 +472,23 @@ public class SegmentWriter {
         return valueId;
     }
 
+    private synchronized RecordId writeValueRecord(int length, byte[] data) {
+        checkArgument(length < Segment.MEDIUM_LIMIT);
+        RecordId id;
+        if (length < Segment.SMALL_LIMIT) {
+            id = prepare(RecordType.VALUE, 1 + length);
+            buffer[position++] = (byte) length;
+        } else {
+            id = prepare(RecordType.VALUE, 2 + length);
+            int len = (length - Segment.SMALL_LIMIT) | 0x8000;
+            buffer[position++] = (byte) (len >> 8);
+            buffer[position++] = (byte) len;
+        }
+        System.arraycopy(data, 0, buffer, position, length);
+        position += length;
+        return id;
+    }
+
     /**
      * Writes a block record containing the given block of bytes.
      *
@@ -535,19 +551,47 @@ public class SegmentWriter {
      * @return value record identifier
      */
     public RecordId writeString(String string) {
+        byte[] data;
+
         synchronized (strings) {
             RecordId id = strings.get(string);
-            if (id == null) {
-                byte[] data = string.getBytes(Charsets.UTF_8);
-                try {
-                    id = writeStream(new ByteArrayInputStream(data)).getRecordId();
-                } catch (IOException e) {
-                    throw new IllegalStateException("Unexpected IOException", e);
-                }
+            if (id != null) {
+                return id;
+            }
+
+            data = string.getBytes(Charsets.UTF_8);
+            if (data.length < Segment.MEDIUM_LIMIT) {
+                id = writeValueRecord(data.length, data);
+
+                // only cache short strings to avoid excessive memory use
                 strings.put(string, id);
+
+                return id;
             }
-            return id;
         }
+
+        int pos = 0;
+        List<RecordId> blockIds = newArrayListWithExpectedSize(
+                data.length / BLOCK_SIZE + 1);
+
+        // write as many full bulk segments as possible
+        while (pos + MAX_SEGMENT_SIZE <= data.length) {
+            UUID uuid = newBulkSegmentId();
+            store.writeSegment(uuid, data, pos, MAX_SEGMENT_SIZE);
+            for (int i = 0; i < MAX_SEGMENT_SIZE; i += BLOCK_SIZE) {
+                blockIds.add(new RecordId(uuid, i));
+            }
+            pos += MAX_SEGMENT_SIZE;
+        }
+
+        // inline the remaining data as block records
+        while (pos < data.length) {
+            int len = Math.min(BLOCK_SIZE, data.length - pos);
+            blockIds.add(writeBlock(data, pos, len));
+            pos += len;
+        }
+
+        return writeValueRecord(data.length, writeList(blockIds));
     }
 
     public SegmentBlob writeBlob(Blob blob) throws IOException {
@@ -588,25 +632,12 @@ public class SegmentWriter {
         // Special case for short binaries (up to about 16kB):
         // store them directly as small- or medium-sized value records
         if (n < Segment.MEDIUM_LIMIT) {
-            synchronized (this) {
-                RecordId id;
-                if (n < Segment.SMALL_LIMIT) {
-                    id = prepare(RecordType.VALUE, 1 + n);
-                    buffer[position++] = (byte) n;
-                } else {
-                    id = prepare(RecordType.VALUE, 2 + n);
-                    int len = (n - Segment.SMALL_LIMIT) | 0x8000;
-                    buffer[position++] = (byte) (len >> 8);
-                    buffer[position++] = (byte) len;
-                }
-                System.arraycopy(data, 0, buffer, position, n);
-                position += n;
-                return id;
-            }
+            return writeValueRecord(n, data);
         }
 
         long length = n;
-        List<RecordId> blockIds = newArrayListWithExpectedSize(n / 4096);
+        List<RecordId> blockIds =
+                newArrayListWithExpectedSize(2 * n / BLOCK_SIZE);
 
         // Write the data to bulk segments and collect the list of block ids
         while (n != 0) {

Modified: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/RecordTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/RecordTest.java?rev=1547183&r1=1547182&r2=1547183&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/RecordTest.java
(original)
+++ jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/RecordTest.java
Mon Dec  2 20:44:49 2013
@@ -164,7 +164,7 @@ public class RecordTest {
         RecordId hello = writer.writeString("Hello, World!");
 
         StringBuilder builder = new StringBuilder();
-        for (int i = 0; i < 100000; i++) {
+        for (int i = 0; i < 2 * Segment.MAX_SEGMENT_SIZE + 1000; i++) {
             builder.append((char) ('0' + i % 10));
         }
         RecordId large = writer.writeString(builder.toString());



Mime
View raw message