jackrabbit-oak-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ju...@apache.org
Subject svn commit: r1524548 - in /jackrabbit/oak/trunk: oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/ oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/ oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/f...
Date Wed, 18 Sep 2013 19:52:47 GMT
Author: jukka
Date: Wed Sep 18 19:52:47 2013
New Revision: 1524548

URL: http://svn.apache.org/r1524548
Log:
OAK-1001: SegmentMK: 32bit support for the file backend

Improved configurability of the different TarMK modes

Modified:
    jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentNodeStore.java
    jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentNodeStoreService.java
    jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java
    jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarFile.java
    jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/file/TarFileTest.java
    jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/benchmark/BenchmarkRunner.java
    jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/fixture/OakRepositoryFixture.java

Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentNodeStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentNodeStore.java?rev=1524548&r1=1524547&r2=1524548&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentNodeStore.java
(original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentNodeStore.java
Wed Sep 18 19:52:47 2013
@@ -30,7 +30,6 @@ import org.apache.jackrabbit.oak.spi.com
 import org.apache.jackrabbit.oak.spi.commit.Observer;
 import org.apache.jackrabbit.oak.spi.state.AbstractNodeStore;
 import org.apache.jackrabbit.oak.spi.state.NodeState;
-import org.apache.jackrabbit.oak.spi.state.NodeStoreBranch;
 
 public class SegmentNodeStore extends AbstractNodeStore {
 

Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentNodeStoreService.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentNodeStoreService.java?rev=1524548&r1=1524547&r2=1524548&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentNodeStoreService.java
(original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentNodeStoreService.java
Wed Sep 18 19:52:47 2013
@@ -16,6 +16,7 @@
  */
 package org.apache.jackrabbit.oak.plugins.segment;
 
+import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
 import java.util.Dictionary;
@@ -55,6 +56,12 @@ public class SegmentNodeStoreService ext
     @Property(description="TarMK directory (if unset, use MongoDB)")
     public static final String DIRECTORY = "repository.home";
 
+    @Property(description="TarMK mode (64 for memory mapping, 32 for normal file access)")
+    public static final String MODE = "tarmk.mode";
+
+    @Property(description="TarMK maximum file size")
+    public static final String SIZE = "tarmk.size";
+
     @Property(description="MongoDB host")
     public static final String HOST = "host";
 
@@ -93,8 +100,25 @@ public class SegmentNodeStoreService ext
         String host = lookup(context, HOST);
         if (host == null) {
             String directory = lookup(context, DIRECTORY);
+            if (directory == null) {
+                directory = "tarmk";
+            }
+
+            String mode = lookup(context, MODE);
+            if (mode == null) {
+                mode = System.getProperty(MODE,
+                        System.getProperty("sun.arch.data.model", "32"));
+            }
+
+            String size = lookup(context, SIZE);
+            if (size == null) {
+                size = System.getProperty(SIZE, "268435456"); // 256MB
+            }
+
             mongo = null;
-            store = new FileStore(directory);
+            store = new FileStore(
+                    new File(directory),
+                    Integer.parseInt(size), "64".equals(mode));
         } else {
             int port = Integer.parseInt(String.valueOf(properties.get(PORT)));
             String db = String.valueOf(properties.get(DB));

Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java?rev=1524548&r1=1524547&r2=1524548&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java
(original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java
Wed Sep 18 19:52:47 2013
@@ -42,7 +42,6 @@ import org.apache.jackrabbit.oak.plugins
 import org.apache.jackrabbit.oak.plugins.segment.SegmentStore;
 import org.apache.jackrabbit.oak.plugins.segment.Template;
 import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
-import org.apache.jackrabbit.oak.spi.state.NodeState;
 
 import com.google.common.cache.Cache;
 import com.google.common.cache.CacheBuilder;
@@ -55,12 +54,14 @@ public class FileStore implements Segmen
 
     static final UUID JOURNALS_UUID = new UUID(0, 0);
 
-    private static final int FILE_SIZE = 256 * 1024 * 1024;
-
     private static final String FILE_NAME_FORMAT = "data%05d.tar";
 
     private final File directory;
 
+    private final int maxFileSize;
+
+    private final boolean memoryMapping;
+
     private final LinkedList<TarFile> files = newLinkedList();
 
     private final Map<String, Journal> journals = newHashMap();
@@ -68,15 +69,18 @@ public class FileStore implements Segmen
     private final Cache<UUID, Segment> segments =
             CacheBuilder.newBuilder().maximumSize(1000).build();
 
-    public FileStore(File directory, NodeState root) throws IOException {
+    public FileStore(File directory, int maxFileSize, boolean memoryMapping)
+            throws IOException {
         checkNotNull(directory).mkdirs();
         this.directory = directory;
+        this.maxFileSize = maxFileSize;
+        this.memoryMapping = memoryMapping;
 
         for (int i = 0; true; i++) {
             String name = String.format(FILE_NAME_FORMAT, i);
             File file = new File(directory, name);
             if (file.isFile()) {
-                files.add(new TarFile(file, FILE_SIZE));
+                files.add(new TarFile(file, maxFileSize, memoryMapping));
             } else {
                 break;
             }
@@ -102,19 +106,11 @@ public class FileStore implements Segmen
 
         if (!journals.containsKey("root")) {
             NodeBuilder builder = EMPTY_NODE.builder();
-            builder.setChildNode("root", root);
+            builder.setChildNode("root", EMPTY_NODE);
             journals.put("root", new FileJournal(this, builder.getNodeState()));
         }
     }
 
-    public FileStore(File directory) throws IOException {
-        this(directory, EMPTY_NODE);
-    }
-
-    public FileStore(String directory) throws IOException {
-        this(new File(directory));
-    }
-
     public synchronized void close() {
         for (TarFile file : files) {
             try {
@@ -217,7 +213,8 @@ public class FileStore implements Segmen
         if (files.isEmpty() || !files.getLast().writeEntry(
                 segmentId, buffer, 0, buffer.length)) {
             String name = String.format(FILE_NAME_FORMAT, files.size());
-            TarFile last = new TarFile(new File(directory, name), FILE_SIZE);
+            File file = new File(directory, name);
+            TarFile last = new TarFile(file, maxFileSize, memoryMapping);
             checkState(last.writeEntry(segmentId, buffer, 0, buffer.length));
             files.add(last);
         }

Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarFile.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarFile.java?rev=1524548&r1=1524547&r2=1524548&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarFile.java
(original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarFile.java
Wed Sep 18 19:52:47 2013
@@ -32,11 +32,6 @@ import com.google.common.collect.Immutab
 
 class TarFile {
 
-    private static boolean USE_MEMORY_MAPPING =
-            System.getProperty("tarmk.mmap") != null
-            ? Boolean.getBoolean("tarmk.mmap")
-            : "64".equals(System.getProperty("sun.arch.data.model"));
-
     /** The tar file block size. */
     private static final int BLOCK_SIZE = 512;
 
@@ -59,17 +54,18 @@ class TarFile {
 
     private int position = 0;
 
-    private final int maxLength;
+    private final int maxFileSize;
 
     private volatile Map<UUID, Location> entries;
 
-    TarFile(File file, int maxLength) throws IOException {
+    TarFile(File file, int maxFileSize, boolean memoryMapping)
+            throws IOException {
         long len = file.length();
         checkState(len <= Integer.MAX_VALUE);
-        this.maxLength = Math.max((int) len, maxLength);
+        this.maxFileSize = Math.max((int) len, maxFileSize);
 
-        if (USE_MEMORY_MAPPING) {
-            this.file = new MappedAccess(file, maxLength);
+        if (memoryMapping) {
+            this.file = new MappedAccess(file, this.maxFileSize);
         } else {
             this.file = new RandomAccess(file);
         }
@@ -122,7 +118,7 @@ class TarFile {
 
     synchronized boolean writeEntry(UUID id, byte[] b, int offset, int size)
             throws IOException {
-        if (position + BLOCK_SIZE + size > maxLength) {
+        if (position + BLOCK_SIZE + size > maxFileSize) {
             return false;
         }
 

Modified: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/file/TarFileTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/file/TarFileTest.java?rev=1524548&r1=1524547&r2=1524548&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/file/TarFileTest.java
(original)
+++ jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/file/TarFileTest.java
Wed Sep 18 19:52:47 2013
@@ -44,8 +44,8 @@ public class TarFileTest {
 
     @Test
     public void testOpenClose() throws IOException {
-        TarFile tar = new TarFile(file, 1024);
-        tar.close();
+        new TarFile(file, 1024, true).close();
+        new TarFile(file, 1024, false).close();
     }
 
     @Test
@@ -53,7 +53,7 @@ public class TarFileTest {
         UUID id = UUID.randomUUID();
         byte[] data = "Hello, World!".getBytes(UTF_8);
 
-        TarFile tar = new TarFile(file, 1024);
+        TarFile tar = new TarFile(file, 1024, false);
         try {
             tar.writeEntry(id, data, 0, data.length);
             assertEquals(ByteBuffer.wrap(data), tar.readEntry(id));
@@ -63,7 +63,7 @@ public class TarFileTest {
 
         assertEquals(1024, file.length());
 
-        tar = new TarFile(file, 1024);
+        tar = new TarFile(file, 1024, false);
         try {
             assertEquals(ByteBuffer.wrap(data), tar.readEntry(id));
         } finally {

Modified: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/benchmark/BenchmarkRunner.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/benchmark/BenchmarkRunner.java?rev=1524548&r1=1524547&r2=1524548&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/benchmark/BenchmarkRunner.java
(original)
+++ jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/benchmark/BenchmarkRunner.java
Wed Sep 18 19:52:47 2013
@@ -43,6 +43,9 @@ public class BenchmarkRunner {
                 .withRequiredArg().defaultsTo("localhost");
         OptionSpec<Integer> port = parser.accepts("port", "MongoDB port")
                 .withRequiredArg().ofType(Integer.class).defaultsTo(27017);
+        OptionSpec<Boolean> mmap = parser.accepts("mmap", "TarMK memory mapping")
+                .withOptionalArg().ofType(Boolean.class)
+                .defaultsTo("64".equals(System.getProperty("sun.arch.data.model")));
         OptionSpec<Integer> cache = parser.accepts("cache", "cache size (MB)")
                 .withRequiredArg().ofType(Integer.class).defaultsTo(100);
         OptionSpec<File> wikipedia =
@@ -68,7 +71,8 @@ public class BenchmarkRunner {
                         host.value(options), port.value(options), cacheSize * MB),
                 OakRepositoryFixture.getSegment(
                         host.value(options), port.value(options), cacheSize * MB),
-                OakRepositoryFixture.getTar(base.value(options))
+                OakRepositoryFixture.getTar(
+                        base.value(options), 256 * 1024 * 1024, mmap.value(options))
         };
         Benchmark[] allBenchmarks = new Benchmark[] {
             new LoginTest(),

Modified: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/fixture/OakRepositoryFixture.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/fixture/OakRepositoryFixture.java?rev=1524548&r1=1524547&r2=1524548&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/fixture/OakRepositoryFixture.java
(original)
+++ jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/fixture/OakRepositoryFixture.java
Wed Sep 18 19:52:47 2013
@@ -142,7 +142,9 @@ public abstract class OakRepositoryFixtu
         };
     }
 
-    public static RepositoryFixture getTar(final File base) {
+    public static RepositoryFixture getTar(
+            final File base,
+            final int maxFileSize, final boolean memoryMapping) {
         return new OakRepositoryFixture("Oak-Tar") {
             private SegmentStore[] stores;
             @Override
@@ -150,7 +152,8 @@ public abstract class OakRepositoryFixtu
                 Repository[] cluster = new Repository[n];
                 stores = new FileStore[cluster.length];
                 for (int i = 0; i < cluster.length; i++) {
-                    stores[i] = new FileStore(new File(base, unique));
+                    stores[i] = new FileStore(
+                            new File(base, unique), maxFileSize, memoryMapping);
                     Oak oak = new Oak(new SegmentNodeStore(stores[i]));
                     cluster[i] = new Jcr(oak).createRepository();
                 }



Mime
View raw message