directory-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kayyag...@apache.org
Subject svn commit: r1546080 - in /directory/mavibot/trunk/mavibot/src: main/java/org/apache/directory/mavibot/btree/managed/ main/java/org/apache/directory/mavibot/btree/memory/ main/java/org/apache/directory/mavibot/btree/util/ test/java/org/apache/directory...
Date Wed, 27 Nov 2013 15:57:03 GMT
Author: kayyagari
Date: Wed Nov 27 15:57:03 2013
New Revision: 1546080

URL: http://svn.apache.org/r1546080
Log:
o modified the TupleReaderWriter interface for better control in implementations
o updated the managed BTree builder class to use RecordManager and renamed
o added a test class for managed BTree builder

Added:
    directory/mavibot/trunk/mavibot/src/main/java/org/apache/directory/mavibot/btree/managed/ManagedBTreeBuilder.java
      - copied, changed from r1545944, directory/mavibot/trunk/mavibot/src/main/java/org/apache/directory/mavibot/btree/managed/BTreeBuilder.java
    directory/mavibot/trunk/mavibot/src/test/java/org/apache/directory/mavibot/btree/managed/ManagedBTreeBuilderTest.java
Removed:
    directory/mavibot/trunk/mavibot/src/main/java/org/apache/directory/mavibot/btree/managed/BTreeBuilder.java
Modified:
    directory/mavibot/trunk/mavibot/src/main/java/org/apache/directory/mavibot/btree/managed/BulkDataSorter.java
    directory/mavibot/trunk/mavibot/src/main/java/org/apache/directory/mavibot/btree/memory/BTreeBuilder.java
    directory/mavibot/trunk/mavibot/src/main/java/org/apache/directory/mavibot/btree/memory/BulkDataSorter.java
    directory/mavibot/trunk/mavibot/src/main/java/org/apache/directory/mavibot/btree/util/IntTupleReaderWriter.java
    directory/mavibot/trunk/mavibot/src/main/java/org/apache/directory/mavibot/btree/util/TupleReaderWriter.java
    directory/mavibot/trunk/mavibot/src/test/java/org/apache/directory/mavibot/btree/memory/BulkDataSorterTest.java

Modified: directory/mavibot/trunk/mavibot/src/main/java/org/apache/directory/mavibot/btree/managed/BulkDataSorter.java
URL: http://svn.apache.org/viewvc/directory/mavibot/trunk/mavibot/src/main/java/org/apache/directory/mavibot/btree/managed/BulkDataSorter.java?rev=1546080&r1=1546079&r2=1546080&view=diff
==============================================================================
--- directory/mavibot/trunk/mavibot/src/main/java/org/apache/directory/mavibot/btree/managed/BulkDataSorter.java
(original)
+++ directory/mavibot/trunk/mavibot/src/main/java/org/apache/directory/mavibot/btree/managed/BulkDataSorter.java
Wed Nov 27 15:57:03 2013
@@ -39,7 +39,7 @@ import org.apache.directory.mavibot.btre
 
 
 /**
- * A utility class for sorting a large number of keys before building a BTree using {@link
BTreeBuilder}.
+ * A utility class for sorting a large number of keys before building a BTree using {@link
ManagedBTreeBuilder}.
  *
  * @author <a href="mailto:dev@directory.apache.org">Apache Directory Project</a>
  */
@@ -84,7 +84,7 @@ public class BulkDataSorter<K, V>
 
         DataInputStream in = new DataInputStream( new FileInputStream( dataFile ) );
 
-        while ( ( t = readerWriter.readTuple( in ) ) != null )
+        while ( ( t = readerWriter.readUnsortedTuple( in ) ) != null )
         {
             arr[i++] = t;
 
@@ -117,7 +117,7 @@ public class BulkDataSorter<K, V>
 
         for ( Tuple<K, V> t : arr )
         {
-            readerWriter.writeTuple( t, out );
+            readerWriter.storeSortedTuple( t, out );
         }
 
         out.flush();
@@ -178,7 +178,7 @@ public class BulkDataSorter<K, V>
                 {
                     if ( heads[i] == null )
                     {
-                        heads[i] = readerWriter.readTuple( streams[i] );
+                        heads[i] = readerWriter.readSortedTuple( streams[i] );
                     }
 
                     if ( available == null )
@@ -209,7 +209,7 @@ public class BulkDataSorter<K, V>
                         if ( heads[i] != null )
                         {
                             available = heads[i];
-                            heads[i] = readerWriter.readTuple( streams[i] );
+                            heads[i] = readerWriter.readUnsortedTuple( streams[i] );
                             break;
                         }
                     }

Copied: directory/mavibot/trunk/mavibot/src/main/java/org/apache/directory/mavibot/btree/managed/ManagedBTreeBuilder.java
(from r1545944, directory/mavibot/trunk/mavibot/src/main/java/org/apache/directory/mavibot/btree/managed/BTreeBuilder.java)
URL: http://svn.apache.org/viewvc/directory/mavibot/trunk/mavibot/src/main/java/org/apache/directory/mavibot/btree/managed/ManagedBTreeBuilder.java?p2=directory/mavibot/trunk/mavibot/src/main/java/org/apache/directory/mavibot/btree/managed/ManagedBTreeBuilder.java&p1=directory/mavibot/trunk/mavibot/src/main/java/org/apache/directory/mavibot/btree/managed/BTreeBuilder.java&r1=1545944&r2=1546080&rev=1546080&view=diff
==============================================================================
--- directory/mavibot/trunk/mavibot/src/main/java/org/apache/directory/mavibot/btree/managed/BTreeBuilder.java
(original)
+++ directory/mavibot/trunk/mavibot/src/main/java/org/apache/directory/mavibot/btree/managed/ManagedBTreeBuilder.java
Wed Nov 27 15:57:03 2013
@@ -29,6 +29,7 @@ import static org.apache.directory.mavib
 import java.io.IOException;
 import java.lang.reflect.Array;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Iterator;
 import java.util.List;
 
@@ -41,7 +42,7 @@ import org.apache.directory.mavibot.btre
  *
  * @author <a href="mailto:dev@directory.apache.org">Apache Directory Project</a>
  */
-public class BTreeBuilder<K, V>
+public class ManagedBTreeBuilder<K, V>
 {
     private String name;
 
@@ -51,10 +52,12 @@ public class BTreeBuilder<K, V>
 
     private ElementSerializer<V> valueSerializer;
 
+    private RecordManager rm;
 
-    public BTreeBuilder( String name, int numKeysInNode, ElementSerializer<K> keySerializer,
+    public ManagedBTreeBuilder( RecordManager rm, String name, int numKeysInNode, ElementSerializer<K>
keySerializer,
         ElementSerializer<V> valueSerializer )
     {
+        this.rm = rm;
         this.name = name;
         this.numKeysInNode = numKeysInNode;
         this.keySerializer = keySerializer;
@@ -63,11 +66,13 @@ public class BTreeBuilder<K, V>
 
 
     @SuppressWarnings("unchecked")
-    public BTree<K, V> build( Iterator<Tuple<K, V>> sortedTupleItr ) throws
IOException
+    public BTree<K, V> build( Iterator<Tuple<K, V>> sortedTupleItr ) throws
Exception
     {
         BTree<K, V> btree = new BTree<K, V>( name, keySerializer, valueSerializer
);
         btree.init();
 
+        rm.manage( btree );
+        
         List<Page<K, V>> lstLeaves = new ArrayList<Page<K, V>>();
 
         int totalTupleCount = 0;
@@ -80,7 +85,7 @@ public class BTreeBuilder<K, V>
         while ( sortedTupleItr.hasNext() )
         {
             Tuple<K, V> tuple = sortedTupleItr.next();
-
+            
             setKey( leaf1, leafIndex, tuple.getKey() );
 
             ValueHolder<V> eh = new ValueHolder<V>( btree, btree.getValueSerializer(),
@@ -93,9 +98,14 @@ public class BTreeBuilder<K, V>
             if ( ( totalTupleCount % numKeysInNode ) == 0 )
             {
                 leafIndex = 0;
+                
+                PageHolder<K, V> pageHolder = ( PageHolder<K, V> ) rm.writePage(
btree, leaf1, 1 );
+                
                 leaf1 = createLeaf( btree, 0, numKeysInNode );
                 lstLeaves.add( leaf1 );
             }
+            
+            //TODO build the whole tree in chunks rather than processing *all* leaves at
first
         }
 
         if ( lstLeaves.isEmpty() )
@@ -120,14 +130,25 @@ public class BTreeBuilder<K, V>
                 lastLeaf.values = ( ValueHolder<V>[] ) Array.newInstance( ValueHolder.class,
n );
                 System.arraycopy( values, 0, lastLeaf.values, 0, n );
 
+                PageHolder<K, V> pageHolder = ( PageHolder<K, V> ) rm.writePage(
btree, lastLeaf, 1 );
+
                 break;
             }
         }
 
+        // make sure either one of the root pages is reclaimed, cause when we call rm.manage()
+        // there is already a root page created
         Page<K, V> rootPage = attachNodes( lstLeaves, btree );
 
+        //System.out.println("built rootpage : " + rootPage);
+        btree.setNbElems( totalTupleCount );
+        
+        rm.updateBtreeHeader( btree, ( ( AbstractPage<K, V> ) rootPage ).getOffset()
);
+        
+        rm.addFreePages( btree, Arrays.asList( btree.rootPage ) );
+        
         btree.rootPage = rootPage;
-
+        
         return btree;
     }
 
@@ -164,6 +185,9 @@ public class BTreeBuilder<K, V>
             if ( ( totalNodes % numChildren ) == 0 )
             {
                 i = 0;
+                
+                PageHolder<K, V> pageHolder = ( PageHolder<K, V> ) rm.writePage(
btree, node, 1 );
+
                 node = createNode( btree, 0, numKeysInNode );
                 lstNodes.add( node );
             }
@@ -183,6 +207,8 @@ public class BTreeBuilder<K, V>
                 lastNode.keys = ( KeyHolder[] ) Array.newInstance( KeyHolder.class, n );
                 System.arraycopy( keys, 0, lastNode.keys, 0, n );
 
+                PageHolder<K, V> pageHolder = ( PageHolder<K, V> ) rm.writePage(
btree, lastNode, 1 );
+
                 break;
             }
         }

Modified: directory/mavibot/trunk/mavibot/src/main/java/org/apache/directory/mavibot/btree/memory/BTreeBuilder.java
URL: http://svn.apache.org/viewvc/directory/mavibot/trunk/mavibot/src/main/java/org/apache/directory/mavibot/btree/memory/BTreeBuilder.java?rev=1546080&r1=1546079&r2=1546080&view=diff
==============================================================================
--- directory/mavibot/trunk/mavibot/src/main/java/org/apache/directory/mavibot/btree/memory/BTreeBuilder.java
(original)
+++ directory/mavibot/trunk/mavibot/src/main/java/org/apache/directory/mavibot/btree/memory/BTreeBuilder.java
Wed Nov 27 15:57:03 2013
@@ -125,6 +125,8 @@ public class BTreeBuilder<K, V>
 
         Page<K, V> rootPage = attachNodes( lstLeaves, btree );
 
+        System.out.println("built rootpage : " + rootPage);
+        
         btree.rootPage = rootPage;
 
         return btree;

Modified: directory/mavibot/trunk/mavibot/src/main/java/org/apache/directory/mavibot/btree/memory/BulkDataSorter.java
URL: http://svn.apache.org/viewvc/directory/mavibot/trunk/mavibot/src/main/java/org/apache/directory/mavibot/btree/memory/BulkDataSorter.java?rev=1546080&r1=1546079&r2=1546080&view=diff
==============================================================================
--- directory/mavibot/trunk/mavibot/src/main/java/org/apache/directory/mavibot/btree/memory/BulkDataSorter.java
(original)
+++ directory/mavibot/trunk/mavibot/src/main/java/org/apache/directory/mavibot/btree/memory/BulkDataSorter.java
Wed Nov 27 15:57:03 2013
@@ -84,7 +84,7 @@ public class BulkDataSorter<K, V>
 
         DataInputStream in = new DataInputStream( new FileInputStream( dataFile ) );
 
-        while ( ( t = readerWriter.readTuple( in ) ) != null )
+        while ( ( t = readerWriter.readUnsortedTuple( in ) ) != null )
         {
             arr[i++] = t;
 
@@ -117,7 +117,7 @@ public class BulkDataSorter<K, V>
 
         for ( Tuple<K, V> t : arr )
         {
-            readerWriter.writeTuple( t, out );
+            readerWriter.storeSortedTuple( t, out );
         }
 
         out.flush();
@@ -178,7 +178,7 @@ public class BulkDataSorter<K, V>
                 {
                     if ( heads[i] == null )
                     {
-                        heads[i] = readerWriter.readTuple( streams[i] );
+                        heads[i] = readerWriter.readUnsortedTuple( streams[i] );
                     }
 
                     if ( available == null )
@@ -209,7 +209,7 @@ public class BulkDataSorter<K, V>
                         if ( heads[i] != null )
                         {
                             available = heads[i];
-                            heads[i] = readerWriter.readTuple( streams[i] );
+                            heads[i] = readerWriter.readUnsortedTuple( streams[i] );
                             break;
                         }
                     }

Modified: directory/mavibot/trunk/mavibot/src/main/java/org/apache/directory/mavibot/btree/util/IntTupleReaderWriter.java
URL: http://svn.apache.org/viewvc/directory/mavibot/trunk/mavibot/src/main/java/org/apache/directory/mavibot/btree/util/IntTupleReaderWriter.java?rev=1546080&r1=1546079&r2=1546080&view=diff
==============================================================================
--- directory/mavibot/trunk/mavibot/src/main/java/org/apache/directory/mavibot/btree/util/IntTupleReaderWriter.java
(original)
+++ directory/mavibot/trunk/mavibot/src/main/java/org/apache/directory/mavibot/btree/util/IntTupleReaderWriter.java
Wed Nov 27 15:57:03 2013
@@ -36,7 +36,7 @@ public class IntTupleReaderWriter implem
 {
 
     @Override
-    public void writeTuple( Tuple<Integer, Integer> t, DataOutputStream out ) throws
IOException
+    public void storeSortedTuple( Tuple<Integer, Integer> t, DataOutputStream out )
throws IOException
     {
         out.writeInt( t.getKey() );
         out.writeInt( t.getValue() );
@@ -44,7 +44,14 @@ public class IntTupleReaderWriter implem
 
 
     @Override
-    public Tuple<Integer, Integer> readTuple( DataInputStream in )
+    public Tuple<Integer, Integer> readSortedTuple( DataInputStream in )
+    {
+        return readUnsortedTuple( in );
+    }
+
+
+    @Override
+    public Tuple<Integer, Integer> readUnsortedTuple( DataInputStream in )
     {
 
         try
@@ -60,6 +67,7 @@ public class IntTupleReaderWriter implem
         }
         catch ( IOException e )
         {
+            e.printStackTrace();
         }
 
         return null;

Modified: directory/mavibot/trunk/mavibot/src/main/java/org/apache/directory/mavibot/btree/util/TupleReaderWriter.java
URL: http://svn.apache.org/viewvc/directory/mavibot/trunk/mavibot/src/main/java/org/apache/directory/mavibot/btree/util/TupleReaderWriter.java?rev=1546080&r1=1546079&r2=1546080&view=diff
==============================================================================
--- directory/mavibot/trunk/mavibot/src/main/java/org/apache/directory/mavibot/btree/util/TupleReaderWriter.java
(original)
+++ directory/mavibot/trunk/mavibot/src/main/java/org/apache/directory/mavibot/btree/util/TupleReaderWriter.java
Wed Nov 27 15:57:03 2013
@@ -34,8 +34,9 @@ import org.apache.directory.mavibot.btre
  */
 public interface TupleReaderWriter<K, V>
 {
-    Tuple<K, V> readTuple( DataInputStream in );
+    Tuple<K, V> readUnsortedTuple( DataInputStream in );
 
-
-    void writeTuple( Tuple<K, V> t, DataOutputStream out ) throws IOException;
+    Tuple<K, V> readSortedTuple( DataInputStream in );
+    
+    void storeSortedTuple( Tuple<K, V> t, DataOutputStream out ) throws IOException;
 }

Added: directory/mavibot/trunk/mavibot/src/test/java/org/apache/directory/mavibot/btree/managed/ManagedBTreeBuilderTest.java
URL: http://svn.apache.org/viewvc/directory/mavibot/trunk/mavibot/src/test/java/org/apache/directory/mavibot/btree/managed/ManagedBTreeBuilderTest.java?rev=1546080&view=auto
==============================================================================
--- directory/mavibot/trunk/mavibot/src/test/java/org/apache/directory/mavibot/btree/managed/ManagedBTreeBuilderTest.java
(added)
+++ directory/mavibot/trunk/mavibot/src/test/java/org/apache/directory/mavibot/btree/managed/ManagedBTreeBuilderTest.java
Wed Nov 27 15:57:03 2013
@@ -0,0 +1,92 @@
+/*
+ *   Licensed to the Apache Software Foundation (ASF) under one
+ *   or more contributor license agreements.  See the NOTICE file
+ *   distributed with this work for additional information
+ *   regarding copyright ownership.  The ASF licenses this file
+ *   to you under the Apache License, Version 2.0 (the
+ *   "License"); you may not use this file except in compliance
+ *   with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   Unless required by applicable law or agreed to in writing,
+ *   software distributed under the License is distributed on an
+ *   "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ *   KIND, either express or implied.  See the License for the
+ *   specific language governing permissions and limitations
+ *   under the License.
+ *
+ */
+
+package org.apache.directory.mavibot.btree.managed;
+
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.directory.mavibot.btree.Tuple;
+import org.apache.directory.mavibot.btree.TupleCursor;
+import org.apache.directory.mavibot.btree.serializer.IntSerializer;
+import org.junit.Test;
+
+
+/**
+ * Test cases for ManagedBTreeBuilder.
+ *
+ * @author <a href="mailto:dev@directory.apache.org">Apache Directory Project</a>
+ */
+public class ManagedBTreeBuilderTest
+{
+
+    @Test
+    public void testManagedBTreeBuilding() throws Exception
+    {
+        List<Tuple<Integer, Integer>> sortedTuple = new ArrayList<Tuple<Integer,
Integer>>();
+        for ( int i = 1; i < 8; i++ )
+        {
+            Tuple<Integer, Integer> t = new Tuple<Integer, Integer>( i, i );
+            sortedTuple.add( t );
+        }
+
+        File file = File.createTempFile( "managedbtreebuilder", ".data" );
+        file.deleteOnExit();
+        
+        RecordManager rm = new RecordManager( file.getAbsolutePath() );
+        
+        IntSerializer ser = new IntSerializer();
+        ManagedBTreeBuilder<Integer, Integer> bb = new ManagedBTreeBuilder<Integer,
Integer>( rm, "master", 4, ser, ser );
+
+        // contains 1, 2, 3, 4, 5, 6, 7
+        BTree<Integer, Integer> btree = bb.build( sortedTuple.iterator() );
+
+        rm.close();
+        
+        rm = new RecordManager( file.getAbsolutePath() );
+        btree = rm.getManagedTree( "master" );
+        
+        assertEquals( 1, btree.rootPage.getNbElems() );
+
+        assertEquals( 7, btree.rootPage.findRightMost().getKey().intValue() );
+
+        assertEquals( 1, btree.rootPage.findLeftMost().getKey().intValue() );
+
+        TupleCursor<Integer, Integer> cursor = btree.browse();
+        int i = 0;
+
+        while ( cursor.hasNext() )
+        {
+            Tuple<Integer, Integer> expected = sortedTuple.get( i++ );
+            Tuple<Integer, Integer> actual = cursor.next();
+            assertEquals( expected.getKey(), actual.getKey() );
+            assertEquals( expected.getValue(), actual.getValue() );
+        }
+
+        cursor.close();
+        btree.close();
+    }
+
+}

Modified: directory/mavibot/trunk/mavibot/src/test/java/org/apache/directory/mavibot/btree/memory/BulkDataSorterTest.java
URL: http://svn.apache.org/viewvc/directory/mavibot/trunk/mavibot/src/test/java/org/apache/directory/mavibot/btree/memory/BulkDataSorterTest.java?rev=1546080&r1=1546079&r2=1546080&view=diff
==============================================================================
--- directory/mavibot/trunk/mavibot/src/test/java/org/apache/directory/mavibot/btree/memory/BulkDataSorterTest.java
(original)
+++ directory/mavibot/trunk/mavibot/src/test/java/org/apache/directory/mavibot/btree/memory/BulkDataSorterTest.java
Wed Nov 27 15:57:03 2013
@@ -80,7 +80,7 @@ public class BulkDataSorterTest
 
             arr[i] = t;
 
-            itrw.writeTuple( t, out );
+            itrw.storeSortedTuple( t, out );
         }
 
         out.close();
@@ -131,7 +131,7 @@ public class BulkDataSorterTest
 
             arr[i] = t;
 
-            itrw.writeTuple( t, out );
+            itrw.storeSortedTuple( t, out );
         }
 
         out.close();



Mime
View raw message