directory-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kayyag...@apache.org
Subject svn commit: r1603478 - /directory/apacheds/trunk/mavibot-partition/src/main/java/org/apache/directory/mavibot/btree/MavibotPartitionBuilder.java
Date Wed, 18 Jun 2014 15:19:35 GMT
Author: kayyagari
Date: Wed Jun 18 15:19:34 2014
New Revision: 1603478

URL: http://svn.apache.org/r1603478
Log:
leaves are removed from memory after creating the size equivalent to numKeysInNode

Modified:
    directory/apacheds/trunk/mavibot-partition/src/main/java/org/apache/directory/mavibot/btree/MavibotPartitionBuilder.java

Modified: directory/apacheds/trunk/mavibot-partition/src/main/java/org/apache/directory/mavibot/btree/MavibotPartitionBuilder.java
URL: http://svn.apache.org/viewvc/directory/apacheds/trunk/mavibot-partition/src/main/java/org/apache/directory/mavibot/btree/MavibotPartitionBuilder.java?rev=1603478&r1=1603477&r2=1603478&view=diff
==============================================================================
--- directory/apacheds/trunk/mavibot-partition/src/main/java/org/apache/directory/mavibot/btree/MavibotPartitionBuilder.java
(original)
+++ directory/apacheds/trunk/mavibot-partition/src/main/java/org/apache/directory/mavibot/btree/MavibotPartitionBuilder.java
Wed Jun 18 15:19:34 2014
@@ -181,13 +181,16 @@ public class MavibotPartitionBuilder
             
             if ( ( totalTupleCount % numKeysInNode ) == 0 )
             {
-                if( ( lstLeaves.size() % numKeysInNode ) == 0 )
+                if( ( lstLeaves.size() % ( numKeysInNode + 1 ) ) == 0 )
                 {
-                    System.out.println( "Processed tuples " + totalTupleCount );
+                    //System.out.println( "Processed tuples " + totalTupleCount );
                     cleanLastLeaf( lstLeaves, btree, newRevision );
-                    Page node = attachNodes( lstLeaves, btree );
-                    lstNodes.add( node );
-                    lstLeaves.clear();
+                    if( !lstLeaves.isEmpty() )
+                    {
+                        Page node = attachNodes( lstLeaves, btree );
+                        lstNodes.add( node );
+                        lstLeaves.clear();
+                    }
                 }
                 
                 leafIndex = 0;
@@ -202,9 +205,12 @@ public class MavibotPartitionBuilder
         if( !lstLeaves.isEmpty() )
         {
             cleanLastLeaf( lstLeaves, btree, newRevision );
-            Page node = attachNodes( lstLeaves, btree );
-            lstNodes.add( node );
-            lstLeaves.clear();
+            if( !lstLeaves.isEmpty() )
+            {
+                Page node = attachNodes( lstLeaves, btree );
+                lstNodes.add( node );
+                lstLeaves.clear();
+            }
         }
         
         if ( lstNodes.isEmpty() )
@@ -215,7 +221,8 @@ public class MavibotPartitionBuilder
         // make sure either one of the root pages is reclaimed, cause when we call rm.manage()
         // there is already a root page created
         Page rootPage = attachNodes( lstNodes, btree );
-
+        lstNodes.clear();
+        
         Page oldRoot = btree.getRootPage();
         
         //System.out.println("built rootpage : " + rootPage);
@@ -253,6 +260,14 @@ public class MavibotPartitionBuilder
         
         // remove null keys and values from the last leaf and resize
         PersistedLeaf lastLeaf = ( PersistedLeaf ) lstLeaves.get( lstLeaves.size() - 1 );
+        
+        if ( lastLeaf.keys[0] == null )
+        {
+            lstLeaves.remove( lastLeaf );
+            System.out.println( "removed last leaf" );
+            return;
+        }
+        
         for ( int i = 0; i < lastLeaf.nbElems; i++ )
         {
             if ( lastLeaf.keys[i] == null )
@@ -318,6 +333,13 @@ public class MavibotPartitionBuilder
         // remove null keys and values from the last node and resize
         AbstractPage lastNode = ( AbstractPage ) lstNodes.get( lstNodes.size() - 1 );
 
+        if ( lastNode.keys[0] == null )
+        {
+            lstNodes.remove( lastNode );
+            System.out.println( "removed last node" );
+            return attachNodes( lstNodes, btree );
+        }
+
         for ( int j = 0; j < lastNode.nbElems; j++ )
         {
             if ( lastNode.keys[j] == null )
@@ -339,14 +361,10 @@ public class MavibotPartitionBuilder
     }
 
 
-    public void calcLevels()
+    private static void calcLevels( int totalKeys, int numKeysPerPage )
     {
         int numLevels = 0;
 
-        int numKeysPerPage = 16;
-
-        int totalKeys = 10000000;
-
         while ( totalKeys > 1 )
         {
             if ( numLevels > 0 )
@@ -364,7 +382,14 @@ public class MavibotPartitionBuilder
                 totalKeys += 1;
             }
 
-            System.out.println( "Total keys " + totalKeys );
+            if ( numLevels == 0 )
+            {
+                System.out.println( "Total Leaves " + totalKeys );
+            }
+            else
+            {
+                System.out.println( "Total Nodes " + totalKeys );
+            }
 
             numLevels++;
         }
@@ -533,6 +558,8 @@ public class MavibotPartitionBuilder
             final Attribute createdTime = new DefaultAttribute( atCreatedTime, DateUtils.getGeneralizedTime()
);
             final Attribute entryCsn = new DefaultAttribute( atCsn, csnFactory.newInstance().toString()
);
             
+            final Tuple t = new Tuple();
+            
             @Override
             public boolean hasNext()
             {
@@ -544,7 +571,6 @@ public class MavibotPartitionBuilder
             public Tuple<String, Entry> next()
             {
                 DnTuple dt = itr.next();
-                Tuple t = new Tuple();
                 t.setKey( dt.getId() );
 
                 try
@@ -554,6 +580,7 @@ public class MavibotPartitionBuilder
                     raf.readFully( data, 0, data.length );
 
                     Entry entry = lar.parseLdifEntry( Strings.utf8ToString( data ) ).getEntry();
+                    data = null;
 
                     entry.add( atEntryUUID, dt.getId() );
                     entry.add( atEntryParentID, dt.getParentId() );
@@ -1202,6 +1229,8 @@ public class MavibotPartitionBuilder
 
     public static void main( String[] args ) throws Exception
     {
+        calcLevels( 50002, 16 );
+        
         File outDir = new File( "/tmp/builder" );
         
         if( outDir.exists() )
@@ -1209,7 +1238,7 @@ public class MavibotPartitionBuilder
             FileUtils.deleteDirectory( outDir );
         }
         
-        File file = new File( "/Users/dbugger/other-projects/slamd~svn/trunk/slamd/package/slamd/tools/MakeLDIF/50k-users.ldif"
);
+        File file = new File( "/Users/dbugger/other-projects/slamd~svn/trunk/slamd/package/slamd/tools/MakeLDIF/30k-users.ldif"
);
 
         //file = new File( "/tmp/builder-test.ldif" );
 //        InputStream in = MavibotPartitionBuilder.class.getClassLoader().getResourceAsStream(
"builder-test.ldif" );



Mime
View raw message