jackrabbit-oak-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From thom...@apache.org
Subject svn commit: r1447234 - in /jackrabbit/oak/trunk/oak-mongomk/src: main/java/org/apache/jackrabbit/mongomk/prototype/ test/java/org/apache/jackrabbit/mongomk/prototype/
Date Mon, 18 Feb 2013 12:59:42 GMT
Author: thomasm
Date: Mon Feb 18 12:59:42 2013
New Revision: 1447234

URL: http://svn.apache.org/r1447234
Log:
OAK-619 Lock-free MongoMK implementation (sharding setup, test, docs)

Added:
    jackrabbit/oak/trunk/oak-mongomk/src/test/java/org/apache/jackrabbit/mongomk/prototype/BlobTest.java
    jackrabbit/oak/trunk/oak-mongomk/src/test/java/org/apache/jackrabbit/mongomk/prototype/RandomStream.java
    jackrabbit/oak/trunk/oak-mongomk/src/test/java/org/apache/jackrabbit/mongomk/prototype/setup.txt
Modified:
    jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/MongoMK.java
    jackrabbit/oak/trunk/oak-mongomk/src/test/java/org/apache/jackrabbit/mongomk/prototype/MongoDocumentStoreTest.java

Modified: jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/MongoMK.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/MongoMK.java?rev=1447234&r1=1447233&r2=1447234&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/MongoMK.java
(original)
+++ jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/MongoMK.java
Mon Feb 18 12:59:42 2013
@@ -30,8 +30,11 @@ import org.apache.jackrabbit.mk.blobs.Me
 import org.apache.jackrabbit.mk.json.JsopReader;
 import org.apache.jackrabbit.mk.json.JsopStream;
 import org.apache.jackrabbit.mk.json.JsopTokenizer;
+import org.apache.jackrabbit.mongomk.impl.blob.MongoBlobStore;
 import org.apache.jackrabbit.oak.commons.PathUtils;
 
+import com.mongodb.DB;
+
 /**
  * A MicroKernel implementation that stores the data in a MongoDB.
  */
@@ -92,11 +95,23 @@ public class MongoMK implements MicroKer
     private Thread backgroundThread;
 
     /**
-     * Create a new in-memory MongoMK to be used for testing.
+     * Create a new in-memory MongoMK used for testing.
      */
     public MongoMK() {
         this(new MemoryDocumentStore(), new MemoryBlobStore(), 0);
     }
+    
+    /**
+     * Create a new MongoMK.
+     * 
+     * @param db the MongoDB connection (null for in-memory)
+     * @param clusterId the cluster id (must be unique)
+     */
+    public MongoMK(DB db, int clusterId) {
+        this(db == null ? new MemoryDocumentStore() : new MongoDocumentStore(db),
+                db == null ? new MemoryBlobStore() : new MongoBlobStore(db), 
+                clusterId);
+    }
 
     /**
      * Create a new MongoMK.

Added: jackrabbit/oak/trunk/oak-mongomk/src/test/java/org/apache/jackrabbit/mongomk/prototype/BlobTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-mongomk/src/test/java/org/apache/jackrabbit/mongomk/prototype/BlobTest.java?rev=1447234&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-mongomk/src/test/java/org/apache/jackrabbit/mongomk/prototype/BlobTest.java
(added)
+++ jackrabbit/oak/trunk/oak-mongomk/src/test/java/org/apache/jackrabbit/mongomk/prototype/BlobTest.java
Mon Feb 18 12:59:42 2013
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.mongomk.prototype;
+
+import static org.junit.Assert.assertEquals;
+
+import java.util.ArrayList;
+import java.util.Random;
+
+import org.junit.Test;
+
+import com.mongodb.DB;
+
+/**
+ * Tests the blob store.
+ */
+public class BlobTest {
+
+    // private static boolean MONGO_DB = true;
+    // private static long TOTAL_SIZE = 1 * 1024 * 1024 * 1024;
+    // private static int DOCUMENT_COUNT = 10;
+
+    private static boolean MONGO_DB = false;
+    private static long TOTAL_SIZE = 1 * 1024 * 1024;
+    private static int DOCUMENT_COUNT = 10;
+    
+    DB openMongoConnection() {
+        return MONGO_DB ? MongoUtils.getConnection().getDB() : null;
+    }
+    
+    void dropCollections() {
+        if (MONGO_DB) {
+            MongoUtils.dropCollections(MongoUtils.getConnection().getDB());
+        }
+    }
+
+    @Test
+    public void addBlobs() throws Exception {
+        MongoMK mk = new MongoMK(openMongoConnection(), 0);
+        long blobSize = TOTAL_SIZE / DOCUMENT_COUNT;
+        ArrayList<String> blobIds = new ArrayList<String>();
+        // use a new seed each time, to allow running the test multiple times 
+        Random r = new Random();
+        for (int i = 0; i < DOCUMENT_COUNT; i++) {
+            log("writing " + i + "/" + DOCUMENT_COUNT);
+            String id = mk.write(new RandomStream(blobSize, r.nextInt()));
+            blobIds.add(id);
+        }
+        for (String id : blobIds) {
+            assertEquals(blobSize, mk.getLength(id));
+        }
+        mk.dispose();
+    }
+    
+    private static void log(String s) {
+        if (TOTAL_SIZE < 10 * 1024 * 1024) {
+            return;
+        }
+        System.out.println(s);
+    }
+    
+}

Modified: jackrabbit/oak/trunk/oak-mongomk/src/test/java/org/apache/jackrabbit/mongomk/prototype/MongoDocumentStoreTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-mongomk/src/test/java/org/apache/jackrabbit/mongomk/prototype/MongoDocumentStoreTest.java?rev=1447234&r1=1447233&r2=1447234&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-mongomk/src/test/java/org/apache/jackrabbit/mongomk/prototype/MongoDocumentStoreTest.java
(original)
+++ jackrabbit/oak/trunk/oak-mongomk/src/test/java/org/apache/jackrabbit/mongomk/prototype/MongoDocumentStoreTest.java
Mon Feb 18 12:59:42 2013
@@ -103,7 +103,6 @@ public class MongoDocumentStoreTest {
     }
 
     @Test
-    @Ignore
     public void addLotsOfNodes() throws Exception {
 
         char[] nPrefix = new char[]{'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K',
'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z'};

Added: jackrabbit/oak/trunk/oak-mongomk/src/test/java/org/apache/jackrabbit/mongomk/prototype/RandomStream.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-mongomk/src/test/java/org/apache/jackrabbit/mongomk/prototype/RandomStream.java?rev=1447234&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-mongomk/src/test/java/org/apache/jackrabbit/mongomk/prototype/RandomStream.java
(added)
+++ jackrabbit/oak/trunk/oak-mongomk/src/test/java/org/apache/jackrabbit/mongomk/prototype/RandomStream.java
Mon Feb 18 12:59:42 2013
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.mongomk.prototype;
+
+import java.io.InputStream;
+import java.util.Random;
+
+/**
+ * A pseudo-random stream.
+ */
+public class RandomStream extends InputStream {
+
+    private long pos, size;
+    private Random random;
+
+    RandomStream(long size, int seed) {
+        this.size = size;
+        this.random = new Random(seed);
+    }
+
+    public int read() {
+        byte[] data = new byte[1];
+        int len = read(data, 0, 1);
+        return len <= 0 ? len : data[0] & 255;
+    }
+
+    public int read(byte[] b, int off, int len) {
+        if (pos >= size) {
+            return -1;
+        }
+        len = (int) Math.min(size - pos, len);
+        long end = off + len;
+        while (off < end) {
+            int r = random.nextInt();
+            b[off++] = (byte) (r ^ (r >>> 16));
+        }
+        pos += len;
+        return len;
+    }
+
+}

Added: jackrabbit/oak/trunk/oak-mongomk/src/test/java/org/apache/jackrabbit/mongomk/prototype/setup.txt
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-mongomk/src/test/java/org/apache/jackrabbit/mongomk/prototype/setup.txt?rev=1447234&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-mongomk/src/test/java/org/apache/jackrabbit/mongomk/prototype/setup.txt
(added)
+++ jackrabbit/oak/trunk/oak-mongomk/src/test/java/org/apache/jackrabbit/mongomk/prototype/setup.txt
Mon Feb 18 12:59:42 2013
@@ -0,0 +1,62 @@
+== MongoDB cluster setup ==
+
+This script will create a MongoDB cluster with 3 shards:
+
+=== Stop
+
+./mongo --port=27017
+use admin
+db.shutdownServer()
+exit
+./mongo --port=10001
+use admin
+db.shutdownServer()
+exit
+./mongo --port=10002
+use admin
+db.shutdownServer()
+exit
+./mongo --port=10003
+use admin
+db.shutdownServer()
+exit
+./mongo --port=20001
+use admin
+db.shutdownServer()
+exit
+ps -ef | grep mongo
+
+=== Clean, Init, and Start
+
+rm -rf db
+mkdir db
+mkdir db/shard1
+mkdir db/shard2
+mkdir db/shard3
+mkdir db/config
+./mongod --shardsvr --port=10001 --dbpath db/shard1 --fork --logpath db/shard1.log
+./mongod --shardsvr --port=10002 --dbpath db/shard2 --fork --logpath db/shard2.log
+./mongod --shardsvr --port=10003 --dbpath db/shard3 --fork --logpath db/shard3.log
+./mongod --configsvr --port 20001 --dbpath=db/config --fork --logpath db/config.log
+./mongos --port 27017 --configdb localhost:20001 --fork --logpath db/mongos.log
+./mongo --host localhost --port 27017
+sh.addShard("localhost:10001")
+sh.addShard("localhost:10002")
+sh.addShard("localhost:10003")
+use MongoMKDB
+db.createCollection("nodes")
+db.createCollection("blobs")
+sh.enableSharding("MongoMKDB")
+sh.shardCollection("MongoMKDB.nodes", { "_id": 1 }, true)
+sh.shardCollection("MongoMKDB.blobs", { "_id": 1 }, true)
+exit
+
+=== Other
+
+Display sharding status:
+
+du -h .
+./mongo --host localhost --port 27017
+db.printShardingStatus()
+exit
+



Mime
View raw message