lucene-java-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From va...@apache.org
Subject svn commit: r165674 - in /lucene/java/trunk/contrib/db: ./ src/java/org/apache/lucene/store/db/ src/test/ src/test/org/ src/test/org/apache/ src/test/org/apache/lucene/ src/test/org/apache/lucene/store/ src/test/org/apache/lucene/store/db/
Date Mon, 02 May 2005 20:06:01 GMT
Author: vajda
Date: Mon May  2 13:06:00 2005
New Revision: 165674

URL: http://svn.apache.org/viewcvs?rev=165674&view=rev
Log:
 - reworked store I/O to use new IndexInput and IndexOutput classes
 - reworked store I/O to avoid upstream buffering giving better txn control
 - added DbStoreTest unit test adapted from StoreTest

Added:
    lucene/java/trunk/contrib/db/src/java/org/apache/lucene/store/db/DbIndexInput.java
    lucene/java/trunk/contrib/db/src/java/org/apache/lucene/store/db/DbIndexOutput.java
    lucene/java/trunk/contrib/db/src/test/
    lucene/java/trunk/contrib/db/src/test/org/
    lucene/java/trunk/contrib/db/src/test/org/apache/
    lucene/java/trunk/contrib/db/src/test/org/apache/lucene/
    lucene/java/trunk/contrib/db/src/test/org/apache/lucene/store/
    lucene/java/trunk/contrib/db/src/test/org/apache/lucene/store/db/
    lucene/java/trunk/contrib/db/src/test/org/apache/lucene/store/db/DbStoreTest.java
Removed:
    lucene/java/trunk/contrib/db/src/java/org/apache/lucene/store/db/DbInputStream.java
    lucene/java/trunk/contrib/db/src/java/org/apache/lucene/store/db/DbOutputStream.java
Modified:
    lucene/java/trunk/contrib/db/   (props changed)
    lucene/java/trunk/contrib/db/build.xml
    lucene/java/trunk/contrib/db/src/java/org/apache/lucene/store/db/Block.java
    lucene/java/trunk/contrib/db/src/java/org/apache/lucene/store/db/DbDirectory.java
    lucene/java/trunk/contrib/db/src/java/org/apache/lucene/store/db/File.java

Propchange: lucene/java/trunk/contrib/db/
------------------------------------------------------------------------------
--- svn:ignore (original)
+++ svn:ignore Mon May  2 13:06:00 2005
@@ -1,2 +1 @@
 lib
-

Modified: lucene/java/trunk/contrib/db/build.xml
URL: http://svn.apache.org/viewcvs/lucene/java/trunk/contrib/db/build.xml?rev=165674&r1=165673&r2=165674&view=diff
==============================================================================
--- lucene/java/trunk/contrib/db/build.xml (original)
+++ lucene/java/trunk/contrib/db/build.xml Mon May  2 13:06:00 2005
@@ -1,5 +1,5 @@
 <?xml version="1.0"?>
-<project name="lucene-db" default="default">
+<project name="db" default="default">
 
   <description>
     Lucene DB integration

Modified: lucene/java/trunk/contrib/db/src/java/org/apache/lucene/store/db/Block.java
URL: http://svn.apache.org/viewcvs/lucene/java/trunk/contrib/db/src/java/org/apache/lucene/store/db/Block.java?rev=165674&r1=165673&r2=165674&view=diff
==============================================================================
--- lucene/java/trunk/contrib/db/src/java/org/apache/lucene/store/db/Block.java (original)
+++ lucene/java/trunk/contrib/db/src/java/org/apache/lucene/store/db/Block.java Mon May  2
13:06:00 2005
@@ -38,7 +38,7 @@
         key = new DatabaseEntry(new byte[fileKey.length + 8]);
         key.setUserBuffer(fileKey.length + 8, true);
 
-        data = new DatabaseEntry(new byte[DbOutputStream.BLOCK_LEN]);
+        data = new DatabaseEntry(new byte[DbIndexOutput.BLOCK_LEN]);
         data.setUserBuffer(data.getSize(), true);
 
         System.arraycopy(fileKey, 0, key.getData(), 0, fileKey.length);
@@ -61,7 +61,7 @@
         byte[] data = key.getData();
         int index = data.length - 8;
 
-        position >>>= DbOutputStream.BLOCK_SHIFT;
+        position >>>= DbIndexOutput.BLOCK_SHIFT;
 
         data[index + 0] = (byte) (0xff & (position >>> 56));
         data[index + 1] = (byte) (0xff & (position >>> 48));
@@ -73,21 +73,21 @@
         data[index + 7] = (byte) (0xff & (position >>>  0));
     }
 
-    protected void get(Db blocks, DbTxn txn, int flags)
+    protected void get(DbDirectory directory)
         throws IOException
     {
         try {
-            blocks.get(txn, key, data, flags);
+            directory.blocks.get(directory.txn, key, data, directory.flags);
         } catch (DatabaseException e) {
             throw new IOException(e.getMessage());
         }
     }
 
-    protected void put(Db blocks, DbTxn txn, int flags)
+    protected void put(DbDirectory directory)
         throws IOException
     {
         try {
-            blocks.put(txn, key, data, 0);
+            directory.blocks.put(directory.txn, key, data, 0);
         } catch (DatabaseException e) {
             throw new IOException(e.getMessage());
         }

Modified: lucene/java/trunk/contrib/db/src/java/org/apache/lucene/store/db/DbDirectory.java
URL: http://svn.apache.org/viewcvs/lucene/java/trunk/contrib/db/src/java/org/apache/lucene/store/db/DbDirectory.java?rev=165674&r1=165673&r2=165674&view=diff
==============================================================================
--- lucene/java/trunk/contrib/db/src/java/org/apache/lucene/store/db/DbDirectory.java (original)
+++ lucene/java/trunk/contrib/db/src/java/org/apache/lucene/store/db/DbDirectory.java Mon
May  2 13:06:00 2005
@@ -19,13 +19,17 @@
 import java.io.IOException;
 import java.io.ByteArrayInputStream;
 import java.io.DataInputStream;
+import java.util.Set;
+import java.util.HashSet;
 import java.util.List;
 import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.Collections;
 
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.Lock;
-import org.apache.lucene.store.OutputStream;
-import org.apache.lucene.store.InputStream;
+import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.store.IndexInput;
 
 import com.sleepycat.db.internal.DbEnv;
 import com.sleepycat.db.internal.Db;
@@ -50,6 +54,7 @@
 
 public class DbDirectory extends Directory {
 
+    protected Set openFiles = Collections.synchronizedSet(new HashSet());
     protected Db files, blocks;
     protected DbTxn txn;
     protected int flags;
@@ -95,24 +100,40 @@
     public void close()
         throws IOException
     {
+        flush();
     }
 
-    public OutputStream createFile(String name)
+    /**
+     * Flush the currently open files. After they have been flushed it is
+     * safe to commit the transaction without closing this DbDirectory
+     * instance first.
+     * @see setTransaction
+     */
+    public void flush()
         throws IOException
     {
-        return new DbOutputStream(files, blocks, txn, flags, name, true);
+        Iterator iterator = openFiles.iterator();
+        
+        while (iterator.hasNext())
+            ((IndexOutput) iterator.next()).flush();
+    }
+
+    public IndexOutput createOutput(String name)
+        throws IOException
+    {
+        return new DbIndexOutput(this, name, true);
     }
 
     public void deleteFile(String name)
         throws IOException
     {
-        new File(name).delete(files, blocks, txn, flags);
+        new File(name).delete(this);
     }
 
     public boolean fileExists(String name)
         throws IOException
     {
-        return new File(name).exists(files, txn, flags);
+        return new File(name).exists(this);
     }
 
     public long fileLength(String name)
@@ -120,7 +141,7 @@
     {
         File file = new File(name);
 
-        if (file.exists(files, txn, flags))
+        if (file.exists(this))
             return file.getLength();
 
         throw new IOException("File does not exist: " + name);
@@ -131,7 +152,7 @@
     {
         File file = new File(name);
 
-        if (file.exists(files, txn, flags))
+        if (file.exists(this))
             return file.getTimeModified();
 
         throw new IOException("File does not exist: " + name);
@@ -184,10 +205,10 @@
         return (String[]) list.toArray(new String[list.size()]);
     }
 
-    public InputStream openFile(String name)
+    public IndexInput openInput(String name)
         throws IOException
     {
-        return new DbInputStream(files, blocks, txn, flags, name);
+        return new DbIndexInput(this, name);
     }
 
     public Lock makeLock(String name)
@@ -198,7 +219,7 @@
     public void renameFile(String from, String to)
         throws IOException
     {
-        new File(from).rename(files, blocks, txn, flags, to);
+        new File(from).rename(this, to);
     }
 
     public void touchFile(String name)
@@ -207,9 +228,31 @@
         File file = new File(name);
         long length = 0L;
 
-        if (file.exists(files, txn, flags))
+        if (file.exists(this))
             length = file.getLength();
 
-        file.modify(files, txn, flags, length, System.currentTimeMillis());
+        file.modify(this, length, System.currentTimeMillis());
+    }
+
+    /**
+     * Once a transaction handle was committed it is no longer valid. In
+     * order to continue using this DbDirectory instance after a commit, the
+     * transaction handle has to be replaced.
+     * @param txn the new transaction handle to use
+     */
+    public void setTransaction(Transaction txn)
+    {
+        setTransaction(txn != null ? DbHandleExtractor.getDbTxn(txn) : null);
+    }
+
+    /**
+     * Once a transaction handle was committed it is no longer valid. In
+     * order to continue using this DbDirectory instance after a commit, the
+     * transaction handle has to be replaced.
+     * @param txn the new transaction handle to use
+     */
+    public void setTransaction(DbTxn txn)
+    {
+        this.txn = txn;
     }
 }

Added: lucene/java/trunk/contrib/db/src/java/org/apache/lucene/store/db/DbIndexInput.java
URL: http://svn.apache.org/viewcvs/lucene/java/trunk/contrib/db/src/java/org/apache/lucene/store/db/DbIndexInput.java?rev=165674&view=auto
==============================================================================
--- lucene/java/trunk/contrib/db/src/java/org/apache/lucene/store/db/DbIndexInput.java (added)
+++ lucene/java/trunk/contrib/db/src/java/org/apache/lucene/store/db/DbIndexInput.java Mon
May  2 13:06:00 2005
@@ -0,0 +1,146 @@
+package org.apache.lucene.store.db;
+
+/**
+ * Copyright 2002-2004 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import org.apache.lucene.store.IndexInput;
+
+
+/**
+ * @author Andi Vajda
+ */
+
+public class DbIndexInput extends IndexInput {
+
+    protected long position = 0L, length = 0L;
+    protected DbDirectory directory;
+    protected Block block;
+    protected File file;
+
+    protected DbIndexInput(DbDirectory directory, String name)
+        throws IOException
+    {
+        super();
+
+        this.directory = directory;
+
+        this.file = new File(name);
+        if (!file.exists(directory))
+            throw new IOException("File does not exist: " + name);
+
+        length = file.getLength();
+
+        block = new Block(file);
+        block.get(directory);
+    }
+
+    public Object clone()
+    {
+        try {
+            DbIndexInput clone = (DbIndexInput) super.clone();
+
+            clone.block = new Block(file);
+            clone.block.seek(position);
+            clone.block.get(directory);
+
+            return clone;
+        } catch (IOException e) {
+            throw new RuntimeException(e.getMessage());
+        }
+    }
+
+    public void close()
+        throws IOException
+    {
+    }
+
+    public long length()
+    {
+        return length;
+    }
+
+    public byte readByte()
+        throws IOException
+    {
+        if (position + 1 > length)
+            throw new IOException("Reading past end of file");
+
+        int blockPos = (int) (position++ & DbIndexOutput.BLOCK_MASK);
+        byte b = block.getData()[blockPos];
+
+        if (blockPos + 1 == DbIndexOutput.BLOCK_LEN)
+        {
+            block.seek(position);
+            block.get(directory);
+        }
+
+        return b;
+    }
+
+    public void readBytes(byte[] b, int offset, int len)
+        throws IOException
+    {
+        if (position + len > length)
+            throw new IOException("Reading past end of file");
+        else
+        {
+            int blockPos = (int) (position & DbIndexOutput.BLOCK_MASK);
+
+            while (blockPos + len >= DbIndexOutput.BLOCK_LEN) {
+                int blockLen = DbIndexOutput.BLOCK_LEN - blockPos;
+
+                System.arraycopy(block.getData(), blockPos,
+                                 b, offset, blockLen);
+
+                len -= blockLen;
+                offset += blockLen;
+                position += blockLen;
+
+                block.seek(position);
+                block.get(directory);
+                blockPos = 0;
+            }
+
+            if (len > 0)
+            {
+                System.arraycopy(block.getData(), blockPos, b, offset, len);
+                position += len;
+            }
+        }
+    }
+
+    public void seek(long pos)
+        throws IOException
+    {
+        if (pos > length)
+            throw new IOException("seeking past end of file");
+
+        if ((pos >>> DbIndexOutput.BLOCK_SHIFT) !=
+            (position >>> DbIndexOutput.BLOCK_SHIFT))
+        {
+            block.seek(pos);
+            block.get(directory);
+        }
+
+        position = pos;
+    }
+
+    public long getFilePointer()
+    {
+        return position;
+    }
+}

Added: lucene/java/trunk/contrib/db/src/java/org/apache/lucene/store/db/DbIndexOutput.java
URL: http://svn.apache.org/viewcvs/lucene/java/trunk/contrib/db/src/java/org/apache/lucene/store/db/DbIndexOutput.java?rev=165674&view=auto
==============================================================================
--- lucene/java/trunk/contrib/db/src/java/org/apache/lucene/store/db/DbIndexOutput.java (added)
+++ lucene/java/trunk/contrib/db/src/java/org/apache/lucene/store/db/DbIndexOutput.java Mon
May  2 13:06:00 2005
@@ -0,0 +1,151 @@
+package org.apache.lucene.store.db;
+
+/**
+ * Copyright 2002-2004 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import org.apache.lucene.store.IndexOutput;
+
+
+/**
+ * @author Andi Vajda
+ */
+
+public class DbIndexOutput extends IndexOutput {
+
+    /**
+     * The size of data blocks, currently 16k (2^14), is determined by this
+     * constant.
+     */
+    static public final int BLOCK_SHIFT = 14;
+    static public final int BLOCK_LEN = 1 << BLOCK_SHIFT;
+    static public final int BLOCK_MASK = BLOCK_LEN - 1;
+
+    protected long position = 0L, length = 0L;
+    protected DbDirectory directory;
+    protected Block block;
+    protected File file;
+
+    protected DbIndexOutput(DbDirectory directory, String name, boolean create)
+        throws IOException
+    {
+        super();
+
+        this.directory = directory;
+
+        file = new File(directory, name, create);
+        block = new Block(file);
+        length = file.getLength();
+
+        seek(length);
+        block.get(directory);
+
+        directory.openFiles.add(this);
+    }
+
+    public void close()
+        throws IOException
+    {
+        flush();
+        file.modify(directory, length, System.currentTimeMillis());
+
+        directory.openFiles.remove(this);
+    }
+
+    public void flush()
+        throws IOException
+    {
+        if (length > 0)
+            block.put(directory);
+    }
+
+    public void writeByte(byte b)
+        throws IOException
+    {
+        int blockPos = (int) (position++ & BLOCK_MASK);
+
+        block.getData()[blockPos] = b;
+
+        if (blockPos + 1 == BLOCK_LEN)
+        {
+            block.put(directory);
+            block.seek(position);
+            block.get(directory);
+        }
+
+        if (position > length)
+            length = position;
+    }
+
+    public void writeBytes(byte[] b, int len)
+        throws IOException
+    {
+        int blockPos = (int) (position & BLOCK_MASK);
+        int offset = 0;
+
+        while (blockPos + len >= BLOCK_LEN) {
+            int blockLen = BLOCK_LEN - blockPos;
+
+            System.arraycopy(b, offset, block.getData(), blockPos, blockLen);
+            block.put(directory);
+
+            len -= blockLen;
+            offset += blockLen;
+            position += blockLen;
+
+            block.seek(position);
+            block.get(directory);
+            blockPos = 0;
+        }
+
+        if (len > 0)
+        {
+            System.arraycopy(b, offset, block.getData(), blockPos, len);
+            position += len;
+        }
+
+        if (position > length)
+            length = position;
+    }
+
+    public long length()
+        throws IOException
+    {
+        return length;
+    }
+
+    public void seek(long pos)
+        throws IOException
+    {
+        if (pos > length)
+            throw new IOException("seeking past end of file");
+
+        if ((pos >>> BLOCK_SHIFT) == (position >>> BLOCK_SHIFT))
+            position = pos;
+        else
+        {
+            block.put(directory);
+            block.seek(pos);
+            block.get(directory);
+            position = pos;
+        }
+    }
+
+    public long getFilePointer()
+    {
+        return position;
+    }
+}

Modified: lucene/java/trunk/contrib/db/src/java/org/apache/lucene/store/db/File.java
URL: http://svn.apache.org/viewcvs/lucene/java/trunk/contrib/db/src/java/org/apache/lucene/store/db/File.java?rev=165674&r1=165673&r2=165674&view=diff
==============================================================================
--- lucene/java/trunk/contrib/db/src/java/org/apache/lucene/store/db/File.java (original)
+++ lucene/java/trunk/contrib/db/src/java/org/apache/lucene/store/db/File.java Mon May  2
13:06:00 2005
@@ -52,13 +52,12 @@
         data.setUserBuffer(data.getSize(), true);
     }
 
-    protected File(Db files, Db blocks, DbTxn txn, int flags,
-                   String name, boolean create)
+    protected File(DbDirectory directory, String name, boolean create)
         throws IOException
     {
         this(name);
 
-        if (!exists(files, txn, flags))
+        if (!exists(directory))
         {
             if (!create)
                 throw new IOException("File does not exist: " + name);
@@ -66,6 +65,9 @@
             {
                 DatabaseEntry key = new DatabaseEntry(new byte[24]);
                 DatabaseEntry data = new DatabaseEntry(null);
+                Db blocks = directory.blocks;
+                DbTxn txn = directory.txn;
+                int flags = directory.flags;
 
                 key.setUserBuffer(24, true);
                 data.setPartial(true);
@@ -131,9 +133,13 @@
         return timeModified;
     }
 
-    protected boolean exists(Db files, DbTxn txn, int flags)
+    protected boolean exists(DbDirectory directory)
         throws IOException
     {
+        Db files = directory.files;
+        DbTxn txn = directory.txn;
+        int flags = directory.flags;
+
         try {
             if (files.get(txn, key, data, flags) == DbConstants.DB_NOTFOUND)
                 return false;
@@ -155,12 +161,13 @@
         return true;
     }
 
-    protected void modify(Db files, DbTxn txn, int flags,
-                          long length, long timeModified)
+    protected void modify(DbDirectory directory, long length, long timeModified)
         throws IOException
     {
         ByteArrayOutputStream buffer = new ByteArrayOutputStream(32);
         DataOutputStream out = new DataOutputStream(buffer);
+        Db files = directory.files;
+        DbTxn txn = directory.txn;
 
         out.writeLong(length);
         out.writeLong(timeModified);
@@ -179,10 +186,10 @@
         this.timeModified = timeModified;
     }
 
-    protected void delete(Db files, Db blocks, DbTxn txn, int flags)
+    protected void delete(DbDirectory directory)
         throws IOException
     {
-        if (!exists(files, txn, flags))
+        if (!exists(directory))
             throw new IOException("File does not exist: " + getName());
 
         Dbc cursor = null;
@@ -194,6 +201,10 @@
                 byte[] cursorBytes = new byte[ulen];
                 DatabaseEntry cursorKey = new DatabaseEntry(cursorBytes);
                 DatabaseEntry cursorData = new DatabaseEntry(null);
+                Db files = directory.files;
+                Db blocks = directory.blocks;
+                DbTxn txn = directory.txn;
+                int flags = directory.flags;
 
                 System.arraycopy(bytes, 0, cursorBytes, 0, bytes.length);
                 cursorKey.setUserBuffer(ulen, true);
@@ -226,19 +237,21 @@
         }
     }
 
-    protected void rename(Db files, Db blocks, DbTxn txn, int flags,
-                          String name)
+    protected void rename(DbDirectory directory, String name)
         throws IOException
     {
-        if (!exists(files, txn, flags))
+        if (!exists(directory))
             throw new IOException("File does not exist: " + getName());
 
         File newFile = new File(name);
 
-        if (newFile.exists(files, txn, flags))
-            newFile.delete(files, blocks, txn, flags);
+        if (newFile.exists(directory))
+            newFile.delete(directory);
 
         try {
+            Db files = directory.files;
+            DbTxn txn = directory.txn;
+
             files.del(txn, key, 0);
             setName(name);
             files.put(txn, key, data, 0);

Added: lucene/java/trunk/contrib/db/src/test/org/apache/lucene/store/db/DbStoreTest.java
URL: http://svn.apache.org/viewcvs/lucene/java/trunk/contrib/db/src/test/org/apache/lucene/store/db/DbStoreTest.java?rev=165674&view=auto
==============================================================================
--- lucene/java/trunk/contrib/db/src/test/org/apache/lucene/store/db/DbStoreTest.java (added)
+++ lucene/java/trunk/contrib/db/src/test/org/apache/lucene/store/db/DbStoreTest.java Mon
May  2 13:06:00 2005
@@ -0,0 +1,412 @@
+package org.apache.lucene.store.db;
+
+/**
+ * Copyright 2005 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.util.Date;
+import java.util.Random;
+import java.util.Arrays;
+
+import java.io.File;
+import java.io.IOException;
+
+import junit.framework.TestCase;
+
+import com.sleepycat.db.EnvironmentConfig;
+import com.sleepycat.db.Environment;
+import com.sleepycat.db.Transaction;
+import com.sleepycat.db.Database;
+import com.sleepycat.db.DatabaseConfig;
+import com.sleepycat.db.DatabaseType;
+import com.sleepycat.db.DatabaseException;
+
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.IndexOutput;
+
+/**
+ * Tests {@link DbDirectory}.
+ *
+ * Adapted from org.apache.lucene.StoreTest with larger files and random bytes.
+ * @author Andi Vajda
+ */
+public class DbStoreTest extends TestCase {
+    protected File dbHome = new File("index");
+    protected Environment env;
+    protected Database index, blocks;
+    
+    public void setUp()
+        throws Exception
+    {
+        if (!dbHome.exists())
+            dbHome.mkdir();
+        else
+        {
+            File[] files = dbHome.listFiles();
+
+            for (int i = 0; i < files.length; i++) {
+                String name = files[i].getName();
+                if (name.startsWith("__") || name.startsWith("log."))
+                    files[i].delete();
+            }
+        }
+
+        EnvironmentConfig envConfig = new EnvironmentConfig();
+        DatabaseConfig dbConfig = new DatabaseConfig();
+
+        envConfig.setTransactional(true);
+        envConfig.setInitializeCache(true);
+        envConfig.setInitializeLocking(true);
+        envConfig.setInitializeLogging(true);
+        envConfig.setAllowCreate(true);
+        envConfig.setThreaded(true);
+        dbConfig.setAllowCreate(true);
+        dbConfig.setType(DatabaseType.BTREE);
+
+        env = new Environment(dbHome, envConfig);
+
+        Transaction txn = null;
+
+        try {
+            txn = env.beginTransaction(null, null);
+            index = env.openDatabase(txn, "__index__", null, dbConfig);
+            blocks = env.openDatabase(txn, "__blocks__", null, dbConfig);
+        } catch (DatabaseException e) {
+            if (txn != null)
+            {
+                txn.abort();
+                txn = null;
+            }
+            index = null;
+            blocks = null;
+            throw e;
+        } finally {
+            if (txn != null)
+                txn.commit();
+            txn = null;
+        }
+    }
+
+    public void tearDown()
+        throws Exception
+    {
+        if (index != null)
+            index.close();
+        if (blocks != null)
+            blocks.close();
+        if (env != null)
+            env.close();
+    }
+
+    public void testBytes()
+        throws Exception
+    {
+        final int count = 250;
+        final int LENGTH_MASK = 0xffff;
+
+        Random gen = new Random(1251971);
+        int totalLength = 0;
+        int duration;
+        Date end;
+    
+        Date veryStart = new Date();
+        Date start = new Date();
+        Transaction txn = null;
+        Directory store = null;
+
+        System.out.println("Writing files byte by byte");
+
+        try {
+            txn = env.beginTransaction(null, null);
+            store = new DbDirectory(txn, index, blocks);
+
+            for (int i = 0; i < count; i++) {
+                String name = i + ".dat";
+                int length = gen.nextInt() & LENGTH_MASK;
+                IndexOutput file = store.createOutput(name);
+
+                totalLength += length;
+
+                for (int j = 0; j < length; j++) {
+                    byte b = (byte)(gen.nextInt() & 0x7F);
+                    file.writeByte(b);
+                }
+      
+                file.close();
+            }
+        } catch (IOException e) {
+            txn.abort();
+            txn = null;
+            throw e;
+        } catch (DatabaseException e) {
+            if (txn != null)
+            {
+                txn.abort();
+                txn = null;
+            }
+            throw e;
+        } finally {
+            if (txn != null)
+                txn.commit();
+
+            store.close();
+        }
+
+        end = new Date();
+
+        duration = (int) (end.getTime() - start.getTime());
+        System.out.print(duration);
+        System.out.print(" total milliseconds to create, ");
+        System.out.print(totalLength / duration);
+        System.out.println(" kb/s");
+
+        try {
+            txn = env.beginTransaction(null, null);
+            store = new DbDirectory(txn, index, blocks);
+
+            gen = new Random(1251971);
+            start = new Date();
+
+            for (int i = 0; i < count; i++) {
+                String name = i + ".dat";
+                int length = gen.nextInt() & LENGTH_MASK;
+                IndexInput file = store.openInput(name);
+
+                if (file.length() != length)
+                    throw new Exception("length incorrect");
+
+                for (int j = 0; j < length; j++) {
+                    byte b = (byte)(gen.nextInt() & 0x7F);
+
+                    if (file.readByte() != b)
+                        throw new Exception("contents incorrect");
+                }
+
+                file.close();
+            }
+        } catch (IOException e) {
+            txn.abort();
+            txn = null;
+            throw e;
+        } catch (DatabaseException e) {
+            if (txn != null)
+            {
+                txn.abort();
+                txn = null;
+            }
+            throw e;
+        } finally {
+            if (txn != null)
+                txn.commit();
+
+            store.close();
+        }
+
+        end = new Date();
+
+        duration = (int) (end.getTime() - start.getTime());
+        System.out.print(duration);
+        System.out.print(" total milliseconds to read, ");
+        System.out.print(totalLength / duration);
+        System.out.println(" kb/s");
+
+        try {
+            txn = env.beginTransaction(null, null);
+            store = new DbDirectory(txn, index, blocks);
+
+            gen = new Random(1251971);
+            start = new Date();
+
+            for (int i = 0; i < count; i++) {
+                String name = i + ".dat";
+                store.deleteFile(name);
+            }
+        } catch (IOException e) {
+            txn.abort();
+            txn = null;
+            throw e;
+        } catch (DatabaseException e) {
+            if (txn != null)
+            {
+                txn.abort();
+                txn = null;
+            }
+            throw e;
+        } finally {
+            if (txn != null)
+                txn.commit();
+
+            store.close();
+        }
+
+        end = new Date();
+
+        System.out.print(end.getTime() - start.getTime());
+        System.out.println(" total milliseconds to delete");
+
+        System.out.print(end.getTime() - veryStart.getTime());
+        System.out.println(" total milliseconds");
+    }
+
+    public void testArrays()
+        throws Exception
+    {
+        final int count = 250;
+        final int LENGTH_MASK = 0xffff;
+
+        Random gen = new Random(1251971);
+        int totalLength = 0;
+        int duration;
+        Date end;
+    
+        Date veryStart = new Date();
+        Date start = new Date();
+        Transaction txn = null;
+        Directory store = null;
+
+        System.out.println("Writing files as one byte array");
+
+        try {
+            txn = env.beginTransaction(null, null);
+            store = new DbDirectory(txn, index, blocks);
+
+            for (int i = 0; i < count; i++) {
+                String name = i + ".dat";
+                int length = gen.nextInt() & LENGTH_MASK;
+                IndexOutput file = store.createOutput(name);
+                byte[] data = new byte[length];
+
+                totalLength += length;
+                gen.nextBytes(data);
+                file.writeBytes(data, length);
+      
+                file.close();
+            }
+        } catch (IOException e) {
+            txn.abort();
+            txn = null;
+            throw e;
+        } catch (DatabaseException e) {
+            if (txn != null)
+            {
+                txn.abort();
+                txn = null;
+            }
+            throw e;
+        } finally {
+            if (txn != null)
+                txn.commit();
+
+            store.close();
+        }
+
+        end = new Date();
+
+        duration = (int) (end.getTime() - start.getTime());
+        System.out.print(duration);
+        System.out.print(" total milliseconds to create, ");
+        System.out.print(totalLength / duration);
+        System.out.println(" kb/s");
+
+        try {
+            txn = env.beginTransaction(null, null);
+            store = new DbDirectory(txn, index, blocks);
+
+            gen = new Random(1251971);
+            start = new Date();
+
+            for (int i = 0; i < count; i++) {
+                String name = i + ".dat";
+                int length = gen.nextInt() & LENGTH_MASK;
+                IndexInput file = store.openInput(name);
+                
+                if (file.length() != length)
+                    throw new Exception("length incorrect");
+
+                byte[] data = new byte[length];
+                byte[] read = new byte[length];
+                gen.nextBytes(data);
+                file.readBytes(read, 0, length);
+
+                if (!Arrays.equals(data, read))
+                    throw new Exception("contents incorrect");
+
+                file.close();
+            }
+        } catch (IOException e) {
+            txn.abort();
+            txn = null;
+            throw e;
+        } catch (DatabaseException e) {
+            if (txn != null)
+            {
+                txn.abort();
+                txn = null;
+            }
+            throw e;
+        } finally {
+            if (txn != null)
+                txn.commit();
+
+            store.close();
+        }
+
+        end = new Date();
+
+        duration = (int) (end.getTime() - start.getTime());
+        System.out.print(duration);
+        System.out.print(" total milliseconds to read, ");
+        System.out.print(totalLength / duration);
+        System.out.println(" kb/s");
+
+        try {
+            txn = env.beginTransaction(null, null);
+            store = new DbDirectory(txn, index, blocks);
+
+            gen = new Random(1251971);
+            start = new Date();
+
+            for (int i = 0; i < count; i++) {
+                String name = i + ".dat";
+                store.deleteFile(name);
+            }
+        } catch (IOException e) {
+            txn.abort();
+            txn = null;
+            throw e;
+        } catch (DatabaseException e) {
+            if (txn != null)
+            {
+                txn.abort();
+                txn = null;
+            }
+            throw e;
+        } finally {
+            if (txn != null)
+                txn.commit();
+
+            store.close();
+        }
+
+        end = new Date();
+
+        System.out.print(end.getTime() - start.getTime());
+        System.out.println(" total milliseconds to delete");
+
+        System.out.print(end.getTime() - veryStart.getTime());
+        System.out.println(" total milliseconds");
+    }
+}



Mime
View raw message