jackrabbit-oak-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ju...@apache.org
Subject svn commit: r1523784 - in /jackrabbit/oak/trunk/oak-core/src: main/java/org/apache/jackrabbit/oak/plugins/segment/file/ test/java/org/apache/jackrabbit/oak/plugins/segment/file/
Date Mon, 16 Sep 2013 20:07:58 GMT
Author: jukka
Date: Mon Sep 16 20:07:58 2013
New Revision: 1523784

URL: http://svn.apache.org/r1523784
Log:
OAK-1001: SegmentMK: 32bit support for the file backend

Add write functionality to the 32bit code, plus a test case

Added:
    jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileAccess.java
      - copied, changed from r1523692, jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/MappedTarFile.java
    jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/MappedAccess.java
      - copied, changed from r1523692, jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/MappedTarFile.java
    jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/RandomAccess.java
      - copied, changed from r1523692, jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/RandomAccessTarFile.java
    jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/file/
    jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/file/TarFileTest.java
Removed:
    jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/MappedTarFile.java
    jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/RandomAccessTarFile.java
Modified:
    jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarFile.java

Copied: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileAccess.java
(from r1523692, jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/MappedTarFile.java)
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileAccess.java?p2=jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileAccess.java&p1=jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/MappedTarFile.java&r1=1523692&r2=1523784&rev=1523784&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/MappedTarFile.java
(original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileAccess.java
Mon Sep 16 20:07:58 2013
@@ -16,37 +16,20 @@
  */
 package org.apache.jackrabbit.oak.plugins.segment.file;
 
-import static java.nio.channels.FileChannel.MapMode.READ_WRITE;
-
-import java.io.File;
 import java.io.IOException;
-import java.io.RandomAccessFile;
 import java.nio.ByteBuffer;
 
-class MappedTarFile extends TarFile {
+interface FileAccess {
+
+    int length() throws IOException;
+
+    ByteBuffer read(int position, int length) throws IOException;
+
+    void write(int position, byte[] b, int offset, int length)
+            throws IOException;
 
-    private final ByteBuffer buffer;
+    void flush() throws IOException;
 
-    MappedTarFile(File file) throws IOException {
-        RandomAccessFile f = new RandomAccessFile(file, "rw");
-        try {
-            buffer = f.getChannel().map(READ_WRITE, 0, f.length());
-        } finally {
-            f.close();
-        }
-    }
-
-    @Override
-    protected int length() {
-        return buffer.limit();
-    }
-
-    @Override
-    protected ByteBuffer read(int position, int length) {
-        ByteBuffer entry = buffer.asReadOnlyBuffer();
-        entry.position(position);
-        entry.limit(position + length);
-        return entry;
-    }
+    void close() throws IOException;
 
-}
+}
\ No newline at end of file

Copied: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/MappedAccess.java
(from r1523692, jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/MappedTarFile.java)
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/MappedAccess.java?p2=jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/MappedAccess.java&p1=jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/MappedTarFile.java&r1=1523692&r2=1523784&rev=1523784&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/MappedTarFile.java
(original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/MappedAccess.java
Mon Sep 16 20:07:58 2013
@@ -22,31 +22,53 @@ import java.io.File;
 import java.io.IOException;
 import java.io.RandomAccessFile;
 import java.nio.ByteBuffer;
+import java.nio.MappedByteBuffer;
 
-class MappedTarFile extends TarFile {
+class MappedAccess implements FileAccess {
 
-    private final ByteBuffer buffer;
+    private final MappedByteBuffer buffer;
 
-    MappedTarFile(File file) throws IOException {
+    MappedAccess(File file, int length) throws IOException {
         RandomAccessFile f = new RandomAccessFile(file, "rw");
         try {
-            buffer = f.getChannel().map(READ_WRITE, 0, f.length());
+            long l = f.length();
+            if (l == 0) { // it's a new file
+                l = length;
+            }
+            buffer = f.getChannel().map(READ_WRITE, 0, l);
         } finally {
             f.close();
         }
     }
 
     @Override
-    protected int length() {
+    public int length() {
         return buffer.limit();
     }
 
     @Override
-    protected ByteBuffer read(int position, int length) {
+    public ByteBuffer read(int position, int length) {
         ByteBuffer entry = buffer.asReadOnlyBuffer();
         entry.position(position);
         entry.limit(position + length);
-        return entry;
+        return entry.slice();
+    }
+
+    @Override
+    public void write(int position, byte[] b, int offset, int length)
+            throws IOException {
+        ByteBuffer entry = buffer.duplicate();
+        entry.position(position);
+        entry.put(b, offset, length);
+    }
+
+    @Override
+    public void flush() {
+        buffer.force();
+    }
+
+    @Override
+    public void close() {
     }
 
 }

Copied: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/RandomAccess.java
(from r1523692, jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/RandomAccessTarFile.java)
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/RandomAccess.java?p2=jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/RandomAccess.java&p1=jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/RandomAccessTarFile.java&r1=1523692&r2=1523784&rev=1523784&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/RandomAccessTarFile.java
(original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/RandomAccess.java
Mon Sep 16 20:07:58 2013
@@ -16,6 +16,7 @@
  */
 package org.apache.jackrabbit.oak.plugins.segment.file;
 
+import static com.google.common.base.Preconditions.checkNotNull;
 import static com.google.common.base.Preconditions.checkState;
 
 import java.io.File;
@@ -23,16 +24,25 @@ import java.io.IOException;
 import java.io.RandomAccessFile;
 import java.nio.ByteBuffer;
 
-class RandomAccessTarFile extends TarFile {
+import javax.annotation.Nonnull;
+
+class RandomAccess implements FileAccess {
 
     private final RandomAccessFile file;
 
-    RandomAccessTarFile(File file) throws IOException {
-        this.file = new RandomAccessFile(file, "rw");
+    RandomAccess(@Nonnull File file) throws IOException {
+        this.file = new RandomAccessFile(checkNotNull(file), "rw");
+    }
+
+    @Override
+    public int length() throws IOException {
+        long length = file.length();
+        checkState(length < Integer.MAX_VALUE);
+        return (int) length;
     }
 
     @Override
-    protected synchronized ByteBuffer read(int position, int length)
+    public synchronized ByteBuffer read(int position, int length)
             throws IOException {
         ByteBuffer entry = ByteBuffer.allocate(length);
         file.seek(position);
@@ -41,14 +51,20 @@ class RandomAccessTarFile extends TarFil
     }
 
     @Override
-    protected int length() throws IOException {
-        long length = file.length();
-        checkState(length < Integer.MAX_VALUE);
-        return (int) length;
+    public synchronized void write(
+            int position, byte[] buffer, int offset, int length)
+            throws IOException {
+        file.seek(position);
+        file.write(buffer, offset, length);
+    }
+
+    @Override
+    public void flush() throws IOException {
+        file.getFD().sync();
     }
 
     @Override
-    void close() throws IOException {
+    public void close() throws IOException {
         file.close();
     }
 

Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarFile.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarFile.java?rev=1523784&r1=1523783&r2=1523784&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarFile.java
(original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarFile.java
Mon Sep 16 20:07:58 2013
@@ -16,44 +16,201 @@
  */
 package org.apache.jackrabbit.oak.plugins.segment.file;
 
-import static java.util.Collections.emptyMap;
+import static com.google.common.base.Charsets.UTF_8;
+import static com.google.common.base.Preconditions.checkState;
 
+import java.io.File;
 import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.util.Map;
 import java.util.UUID;
 
-abstract class TarFile {
+import com.google.common.collect.ImmutableMap;
 
-    protected static class Entry {
+class TarFile {
 
-        int position;
+    private static boolean USE_MEMORY_MAPPING =
+            System.getProperty("tarmk.mmap") != null
+            ? Boolean.getBoolean("tarmk.mmap")
+            : "64".equals(System.getProperty("sun.arch.data.model"));
 
-        int length;
+    /** The tar file block size. */
+    private static final int BLOCK_SIZE = 512;
 
-        Entry(int position, int length) {
-            this.position = position;
-            this.length = length;
+    private static final byte[] ZERO_BYTES = new byte[BLOCK_SIZE];
+
+    private static class Location {
+
+        int offset;
+
+        int size;
+
+        Location(int offset, int size) {
+            this.offset = offset;
+            this.size = size;
         }
 
     }
 
-    private volatile Map<UUID, Entry> entries = emptyMap();
+    private final FileAccess file;
+
+    private int position = 0;
+
+    private final int maxLength;
+
+    private volatile Map<UUID, Location> entries;
+
+    TarFile(File file, int maxLength) throws IOException {
+        long len = file.length();
+        checkState(len <= Integer.MAX_VALUE);
+        this.maxLength = Math.max((int) len, maxLength);
+
+        if (USE_MEMORY_MAPPING) {
+            this.file = new MappedAccess(file, maxLength);
+        } else {
+            this.file = new RandomAccess(file);
+        }
+
+        ImmutableMap.Builder<UUID, Location> builder = ImmutableMap.builder();
+
+        this.position = 0;
+        while (position + BLOCK_SIZE <= len) {
+            // read the tar header block
+            ByteBuffer buffer = this.file.read(position, BLOCK_SIZE);
+            String name = readString(buffer, 100);
+            buffer.position(124);
+            int size = readNumber(buffer, 12);
+            // TODO: verify the checksum, magic, etc.?
+
+            if (name.isEmpty() && size == 0) {
+                break; // no more entries in this file
+            }
+
+            try {
+                UUID id = UUID.fromString(name);
+                builder.put(id, new Location(position + BLOCK_SIZE, size));
+            } catch (IllegalArgumentException e) {
+                throw new IOException("Unexpected tar entry: " + name);
+            }
+
+            position += (1 + (size + BLOCK_SIZE - 1) / BLOCK_SIZE) * BLOCK_SIZE;
+        }
+
+        this.entries = builder.build();
+    }
 
     ByteBuffer readEntry(UUID id) throws IOException {
-        Entry entry = entries.get(id);
-        if (entry != null) {
-            return read(entry.position, entry.length);
+        Location location = entries.get(id);
+        if (location != null) {
+            return file.read(location.offset, location.size);
         } else {
             return null;
         }
     }
 
-    protected abstract int length() throws IOException;
+    synchronized boolean writeEntry(UUID id, byte[] b, int offset, int size)
+            throws IOException {
+        if (position + BLOCK_SIZE + size > maxLength) {
+            return false;
+        }
+
+        byte[] header = new byte[BLOCK_SIZE];
 
-    protected abstract ByteBuffer read(int position, int length)
-            throws IOException;
+        // File name
+        byte[] n = id.toString().getBytes(UTF_8);
+        System.arraycopy(n, 0, header, 0, n.length);
+
+        // File mode
+        System.arraycopy(
+                String.format("%07o", 0400).getBytes(UTF_8), 0,
+                header, 100, 7);
+
+        // User's numeric user ID
+        System.arraycopy(
+                String.format("%07o", 0).getBytes(UTF_8), 0,
+                header, 108, 7);
+
+        // Group's numeric user ID
+        System.arraycopy(
+                String.format("%07o", 0).getBytes(UTF_8), 0,
+                header, 116, 7);
+
+        // File size in bytes (octal basis)
+        System.arraycopy(
+                String.format("%011o", size).getBytes(UTF_8), 0,
+                header, 124, 11);
+
+        // Last modification time in numeric Unix time format (octal)
+        long time = System.currentTimeMillis() / 1000;
+        System.arraycopy(
+                String.format("%011o", time).getBytes(UTF_8), 0,
+                header, 136, 11);
+
+        // Checksum for header record
+        System.arraycopy(
+                new byte[] { ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ' }, 0,
+                header, 148, 8);
+
+        // Type flag
+        header[156] = '0';
+
+        // Compute checksum
+        int checksum = 0;
+        for (int i = 0; i < header.length; i++) {
+            checksum += header[i] & 0xff;
+        }
+        System.arraycopy(
+                String.format("%06o", checksum).getBytes(UTF_8), 0,
+                header, 148, 6);
+        header[154] = 0;
+
+        file.write(position, header, 0, BLOCK_SIZE);
+        position += BLOCK_SIZE;
+
+        file.write(position, b, offset, size);
+        entries = ImmutableMap.<UUID, Location>builder()
+                .putAll(entries)
+                .put(id, new Location(position, size))
+                .build();
+        position += size;
+
+        int padding = BLOCK_SIZE - position % BLOCK_SIZE;
+        if (padding < BLOCK_SIZE) {
+            file.write(position, ZERO_BYTES, 0, padding);
+            position += padding;
+        }
+
+        return true;
+    }
 
     void close() throws IOException {
+        file.flush();
+        file.close();
     }
+
+    private static String readString(ByteBuffer buffer, int fieldSize) {
+        byte[] b = new byte[fieldSize];
+        buffer.get(b);
+        int n = 0;
+        while (n < fieldSize && b[n] != 0) {
+            n++;
+        }
+        return new String(b, 0, n, UTF_8);
+    }
+
+    private static int readNumber(ByteBuffer buffer, int fieldSize) {
+        byte[] b = new byte[fieldSize];
+        buffer.get(b);
+        int number = 0;
+        for (int i = 0; i < fieldSize; i++) {
+            int digit = b[i] & 0xff;
+            if ('0' <= digit && digit <= '7') {
+                number = number * 8 + digit - '0';
+            } else {
+                break;
+            }
+        }
+        return number;
+    }
+
 }

Added: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/file/TarFileTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/file/TarFileTest.java?rev=1523784&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/file/TarFileTest.java
(added)
+++ jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/file/TarFileTest.java
Mon Sep 16 20:07:58 2013
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.oak.plugins.segment.file;
+
+import static com.google.common.base.Charsets.UTF_8;
+import static junit.framework.Assert.assertEquals;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.UUID;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TarFileTest {
+
+    private File file;
+
+    @Before
+    public void setUp() throws IOException {
+        file = File.createTempFile("TarFileTest", ".tar");
+    }
+
+    @After
+    public void tearDown() {
+        // file.delete();
+    }
+
+    @Test
+    public void testOpenClose() throws IOException {
+        TarFile tar = new TarFile(file, 1024);
+        tar.close();
+    }
+
+    @Test
+    public void testWriteAndRead() throws IOException {
+        UUID id = UUID.randomUUID();
+        byte[] data = "Hello, World!".getBytes(UTF_8);
+
+        TarFile tar = new TarFile(file, 1024);
+        try {
+            tar.writeEntry(id, data, 0, data.length);
+            assertEquals(ByteBuffer.wrap(data), tar.readEntry(id));
+        } finally {
+            tar.close();
+        }
+
+        assertEquals(1024, file.length());
+
+        tar = new TarFile(file, 1024);
+        try {
+            assertEquals(ByteBuffer.wrap(data), tar.readEntry(id));
+        } finally {
+            tar.close();
+        }
+    }
+
+}



Mime
View raw message