Return-Path: X-Original-To: apmail-jackrabbit-oak-commits-archive@minotaur.apache.org Delivered-To: apmail-jackrabbit-oak-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 57A771022D for ; Wed, 18 Sep 2013 19:53:15 +0000 (UTC) Received: (qmail 29451 invoked by uid 500); 18 Sep 2013 19:53:15 -0000 Delivered-To: apmail-jackrabbit-oak-commits-archive@jackrabbit.apache.org Received: (qmail 29410 invoked by uid 500); 18 Sep 2013 19:53:11 -0000 Mailing-List: contact oak-commits-help@jackrabbit.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: oak-dev@jackrabbit.apache.org Delivered-To: mailing list oak-commits@jackrabbit.apache.org Received: (qmail 29401 invoked by uid 99); 18 Sep 2013 19:53:10 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 18 Sep 2013 19:53:10 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 18 Sep 2013 19:53:07 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id D586823889E1; Wed, 18 Sep 2013 19:52:47 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1524548 - in /jackrabbit/oak/trunk: oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/ oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/ oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/f... Date: Wed, 18 Sep 2013 19:52:47 -0000 To: oak-commits@jackrabbit.apache.org From: jukka@apache.org X-Mailer: svnmailer-1.0.9 Message-Id: <20130918195247.D586823889E1@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: jukka Date: Wed Sep 18 19:52:47 2013 New Revision: 1524548 URL: http://svn.apache.org/r1524548 Log: OAK-1001: SegmentMK: 32bit support for the file backend Improved configurability of the different TarMK modes Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentNodeStore.java jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentNodeStoreService.java jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarFile.java jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/file/TarFileTest.java jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/benchmark/BenchmarkRunner.java jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/fixture/OakRepositoryFixture.java Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentNodeStore.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentNodeStore.java?rev=1524548&r1=1524547&r2=1524548&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentNodeStore.java (original) +++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentNodeStore.java Wed Sep 18 19:52:47 2013 @@ -30,7 +30,6 @@ import org.apache.jackrabbit.oak.spi.com import org.apache.jackrabbit.oak.spi.commit.Observer; import org.apache.jackrabbit.oak.spi.state.AbstractNodeStore; import org.apache.jackrabbit.oak.spi.state.NodeState; -import org.apache.jackrabbit.oak.spi.state.NodeStoreBranch; public class SegmentNodeStore extends AbstractNodeStore { Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentNodeStoreService.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentNodeStoreService.java?rev=1524548&r1=1524547&r2=1524548&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentNodeStoreService.java (original) +++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentNodeStoreService.java Wed Sep 18 19:52:47 2013 @@ -16,6 +16,7 @@ */ package org.apache.jackrabbit.oak.plugins.segment; +import java.io.File; import java.io.IOException; import java.io.InputStream; import java.util.Dictionary; @@ -55,6 +56,12 @@ public class SegmentNodeStoreService ext @Property(description="TarMK directory (if unset, use MongoDB)") public static final String DIRECTORY = "repository.home"; + @Property(description="TarMK mode (64 for memory mapping, 32 for normal file access)") + public static final String MODE = "tarmk.mode"; + + @Property(description="TarMK maximum file size") + public static final String SIZE = "tarmk.size"; + @Property(description="MongoDB host") public static final String HOST = "host"; @@ -93,8 +100,25 @@ public class SegmentNodeStoreService ext String host = lookup(context, HOST); if (host == null) { String directory = lookup(context, DIRECTORY); + if (directory == null) { + directory = "tarmk"; + } + + String mode = lookup(context, MODE); + if (mode == null) { + mode = System.getProperty(MODE, + System.getProperty("sun.arch.data.model", "32")); + } + + String size = lookup(context, SIZE); + if (size == null) { + size = System.getProperty(SIZE, "268435456"); // 256MB + } + mongo = null; - store = new FileStore(directory); + store = new FileStore( + new File(directory), + Integer.parseInt(size), "64".equals(mode)); } else { int port = Integer.parseInt(String.valueOf(properties.get(PORT))); String db = String.valueOf(properties.get(DB)); Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java?rev=1524548&r1=1524547&r2=1524548&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java (original) +++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java Wed Sep 18 19:52:47 2013 @@ -42,7 +42,6 @@ import org.apache.jackrabbit.oak.plugins import org.apache.jackrabbit.oak.plugins.segment.SegmentStore; import org.apache.jackrabbit.oak.plugins.segment.Template; import org.apache.jackrabbit.oak.spi.state.NodeBuilder; -import org.apache.jackrabbit.oak.spi.state.NodeState; import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; @@ -55,12 +54,14 @@ public class FileStore implements Segmen static final UUID JOURNALS_UUID = new UUID(0, 0); - private static final int FILE_SIZE = 256 * 1024 * 1024; - private static final String FILE_NAME_FORMAT = "data%05d.tar"; private final File directory; + private final int maxFileSize; + + private final boolean memoryMapping; + private final LinkedList files = newLinkedList(); private final Map journals = newHashMap(); @@ -68,15 +69,18 @@ public class FileStore implements Segmen private final Cache segments = CacheBuilder.newBuilder().maximumSize(1000).build(); - public FileStore(File directory, NodeState root) throws IOException { + public FileStore(File directory, int maxFileSize, boolean memoryMapping) + throws IOException { checkNotNull(directory).mkdirs(); this.directory = directory; + this.maxFileSize = maxFileSize; + this.memoryMapping = memoryMapping; for (int i = 0; true; i++) { String name = String.format(FILE_NAME_FORMAT, i); File file = new File(directory, name); if (file.isFile()) { - files.add(new TarFile(file, FILE_SIZE)); + files.add(new TarFile(file, maxFileSize, memoryMapping)); } else { break; } @@ -102,19 +106,11 @@ public class FileStore implements Segmen if (!journals.containsKey("root")) { NodeBuilder builder = EMPTY_NODE.builder(); - builder.setChildNode("root", root); + builder.setChildNode("root", EMPTY_NODE); journals.put("root", new FileJournal(this, builder.getNodeState())); } } - public FileStore(File directory) throws IOException { - this(directory, EMPTY_NODE); - } - - public FileStore(String directory) throws IOException { - this(new File(directory)); - } - public synchronized void close() { for (TarFile file : files) { try { @@ -217,7 +213,8 @@ public class FileStore implements Segmen if (files.isEmpty() || !files.getLast().writeEntry( segmentId, buffer, 0, buffer.length)) { String name = String.format(FILE_NAME_FORMAT, files.size()); - TarFile last = new TarFile(new File(directory, name), FILE_SIZE); + File file = new File(directory, name); + TarFile last = new TarFile(file, maxFileSize, memoryMapping); checkState(last.writeEntry(segmentId, buffer, 0, buffer.length)); files.add(last); } Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarFile.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarFile.java?rev=1524548&r1=1524547&r2=1524548&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarFile.java (original) +++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarFile.java Wed Sep 18 19:52:47 2013 @@ -32,11 +32,6 @@ import com.google.common.collect.Immutab class TarFile { - private static boolean USE_MEMORY_MAPPING = - System.getProperty("tarmk.mmap") != null - ? Boolean.getBoolean("tarmk.mmap") - : "64".equals(System.getProperty("sun.arch.data.model")); - /** The tar file block size. */ private static final int BLOCK_SIZE = 512; @@ -59,17 +54,18 @@ class TarFile { private int position = 0; - private final int maxLength; + private final int maxFileSize; private volatile Map entries; - TarFile(File file, int maxLength) throws IOException { + TarFile(File file, int maxFileSize, boolean memoryMapping) + throws IOException { long len = file.length(); checkState(len <= Integer.MAX_VALUE); - this.maxLength = Math.max((int) len, maxLength); + this.maxFileSize = Math.max((int) len, maxFileSize); - if (USE_MEMORY_MAPPING) { - this.file = new MappedAccess(file, maxLength); + if (memoryMapping) { + this.file = new MappedAccess(file, this.maxFileSize); } else { this.file = new RandomAccess(file); } @@ -122,7 +118,7 @@ class TarFile { synchronized boolean writeEntry(UUID id, byte[] b, int offset, int size) throws IOException { - if (position + BLOCK_SIZE + size > maxLength) { + if (position + BLOCK_SIZE + size > maxFileSize) { return false; } Modified: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/file/TarFileTest.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/file/TarFileTest.java?rev=1524548&r1=1524547&r2=1524548&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/file/TarFileTest.java (original) +++ jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/file/TarFileTest.java Wed Sep 18 19:52:47 2013 @@ -44,8 +44,8 @@ public class TarFileTest { @Test public void testOpenClose() throws IOException { - TarFile tar = new TarFile(file, 1024); - tar.close(); + new TarFile(file, 1024, true).close(); + new TarFile(file, 1024, false).close(); } @Test @@ -53,7 +53,7 @@ public class TarFileTest { UUID id = UUID.randomUUID(); byte[] data = "Hello, World!".getBytes(UTF_8); - TarFile tar = new TarFile(file, 1024); + TarFile tar = new TarFile(file, 1024, false); try { tar.writeEntry(id, data, 0, data.length); assertEquals(ByteBuffer.wrap(data), tar.readEntry(id)); @@ -63,7 +63,7 @@ public class TarFileTest { assertEquals(1024, file.length()); - tar = new TarFile(file, 1024); + tar = new TarFile(file, 1024, false); try { assertEquals(ByteBuffer.wrap(data), tar.readEntry(id)); } finally { Modified: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/benchmark/BenchmarkRunner.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/benchmark/BenchmarkRunner.java?rev=1524548&r1=1524547&r2=1524548&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/benchmark/BenchmarkRunner.java (original) +++ jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/benchmark/BenchmarkRunner.java Wed Sep 18 19:52:47 2013 @@ -43,6 +43,9 @@ public class BenchmarkRunner { .withRequiredArg().defaultsTo("localhost"); OptionSpec port = parser.accepts("port", "MongoDB port") .withRequiredArg().ofType(Integer.class).defaultsTo(27017); + OptionSpec mmap = parser.accepts("mmap", "TarMK memory mapping") + .withOptionalArg().ofType(Boolean.class) + .defaultsTo("64".equals(System.getProperty("sun.arch.data.model"))); OptionSpec cache = parser.accepts("cache", "cache size (MB)") .withRequiredArg().ofType(Integer.class).defaultsTo(100); OptionSpec wikipedia = @@ -68,7 +71,8 @@ public class BenchmarkRunner { host.value(options), port.value(options), cacheSize * MB), OakRepositoryFixture.getSegment( host.value(options), port.value(options), cacheSize * MB), - OakRepositoryFixture.getTar(base.value(options)) + OakRepositoryFixture.getTar( + base.value(options), 256 * 1024 * 1024, mmap.value(options)) }; Benchmark[] allBenchmarks = new Benchmark[] { new LoginTest(), Modified: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/fixture/OakRepositoryFixture.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/fixture/OakRepositoryFixture.java?rev=1524548&r1=1524547&r2=1524548&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/fixture/OakRepositoryFixture.java (original) +++ jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/fixture/OakRepositoryFixture.java Wed Sep 18 19:52:47 2013 @@ -142,7 +142,9 @@ public abstract class OakRepositoryFixtu }; } - public static RepositoryFixture getTar(final File base) { + public static RepositoryFixture getTar( + final File base, + final int maxFileSize, final boolean memoryMapping) { return new OakRepositoryFixture("Oak-Tar") { private SegmentStore[] stores; @Override @@ -150,7 +152,8 @@ public abstract class OakRepositoryFixtu Repository[] cluster = new Repository[n]; stores = new FileStore[cluster.length]; for (int i = 0; i < cluster.length; i++) { - stores[i] = new FileStore(new File(base, unique)); + stores[i] = new FileStore( + new File(base, unique), maxFileSize, memoryMapping); Oak oak = new Oak(new SegmentNodeStore(stores[i])); cluster[i] = new Jcr(oak).createRepository(); }