directory-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kayyag...@apache.org
Subject svn commit: r1604381 - in /directory/apacheds/trunk/mavibot-partition/src/main/java/org/apache/directory/mavibot/btree: MavibotPartitionBuilder.java Option.java
Date Sat, 21 Jun 2014 13:49:57 GMT
Author: kayyagari
Date: Sat Jun 21 13:49:57 2014
New Revision: 1604381

URL: http://svn.apache.org/r1604381
Log:
added CLI to bulk loader

Added:
    directory/apacheds/trunk/mavibot-partition/src/main/java/org/apache/directory/mavibot/btree/Option.java
Modified:
    directory/apacheds/trunk/mavibot-partition/src/main/java/org/apache/directory/mavibot/btree/MavibotPartitionBuilder.java

Modified: directory/apacheds/trunk/mavibot-partition/src/main/java/org/apache/directory/mavibot/btree/MavibotPartitionBuilder.java
URL: http://svn.apache.org/viewvc/directory/apacheds/trunk/mavibot-partition/src/main/java/org/apache/directory/mavibot/btree/MavibotPartitionBuilder.java?rev=1604381&r1=1604380&r2=1604381&view=diff
==============================================================================
--- directory/apacheds/trunk/mavibot-partition/src/main/java/org/apache/directory/mavibot/btree/MavibotPartitionBuilder.java
(original)
+++ directory/apacheds/trunk/mavibot-partition/src/main/java/org/apache/directory/mavibot/btree/MavibotPartitionBuilder.java
Sat Jun 21 13:49:57 2014
@@ -101,7 +101,7 @@ public class MavibotPartitionBuilder
 
     private RecordManager rm;
 
-    private MavibotPartition partition;
+    //private MavibotPartition partition;
 
     private SchemaManager schemaManager;
 
@@ -405,31 +405,6 @@ public class MavibotPartitionBuilder
     }
 
 
-    private void createPartition() throws Exception
-    {
-        DnFactory dnFactory = new DefaultDnFactory( schemaManager, null );
-
-        partition = new MavibotPartition( schemaManager, dnFactory );
-        partition.setId( "builder" );
-        partition.setSuffixDn( suffixDn );
-
-        File dir = new File( outputDir );
-        partition.setPartitionPath( dir.toURI() );
-
-        for( String atName : indexAttributes )
-        {
-            schemaManager.lookupAttributeTypeRegistry( atName );
-            partition.addIndex( new MavibotIndex( atName, false ) );
-        }
-        
-        partition.initialize();
-
-        masterTableName = partition.getMasterTable().getName();
-        
-        rm = partition.getRecordMan();
-    }
-
-
     private Set<DnTuple> getDnTuples() throws Exception
     {
         File file = new File( ldifFile );
@@ -792,11 +767,33 @@ public class MavibotPartitionBuilder
             LOG.info( message );
         }
         
+        MavibotPartition partition = null;
         try
         {
             long partT0 = System.currentTimeMillis();
             System.out.print( "Creating partition..." );
-            createPartition();
+            
+            DnFactory dnFactory = new DefaultDnFactory( schemaManager, null );
+
+            partition = new MavibotPartition( schemaManager, dnFactory );
+            partition.setId( "builder" );
+            partition.setSuffixDn( suffixDn );
+
+            File dir = new File( outputDir );
+            partition.setPartitionPath( dir.toURI() );
+
+            for( String atName : indexAttributes )
+            {
+                schemaManager.lookupAttributeTypeRegistry( atName );
+                partition.addIndex( new MavibotIndex( atName, false ) );
+            }
+            
+            partition.initialize();
+
+            masterTableName = partition.getMasterTable().getName();
+            
+            rm = partition.getRecordMan();
+
             long partT1 = System.currentTimeMillis();
             System.out.println( ", time taken : " + ( partT1 - partT0 ) + "ms" );
         }
@@ -821,15 +818,16 @@ public class MavibotPartitionBuilder
             return;
         }
         
+        Iterator<String> userIndexItr = partition.getUserIndices();
         
         try
         {
-            // the partition must be re-initialized cause we are
+            // the RecordManager must be re-initialized cause we are
             // setting the "values" of leaves to null while building
             // the tree to avoid OOM errors
             partition.destroy();
-            partition.initialize();
-            rm = partition.getRecordMan();
+            
+            rm = new RecordManager( new File( partition.getPartitionPath() ).getAbsolutePath()
);
             
             long rdnT0 = System.currentTimeMillis();
             System.out.print( "Building RDN index." );
@@ -887,7 +885,7 @@ public class MavibotPartitionBuilder
         {
             System.out.print( "Building presence index..." );
             long presenceT0 = System.currentTimeMillis();
-            buildPresenceIndex();
+            buildPresenceIndex( userIndexItr );
             long presenceT1 = System.currentTimeMillis();
             System.out.println( ", time taken : " + ( presenceT1 - presenceT0 ) + "ms" );
         }
@@ -902,10 +900,9 @@ public class MavibotPartitionBuilder
     }
 
     
-    private void buildPresenceIndex() throws Exception
+    private void buildPresenceIndex( Iterator<String> itr ) throws Exception
     {
         Set<String> idxOids = new HashSet<String>();
-        Iterator<String> itr = partition.getUserIndices();
         
         while( itr.hasNext() )
         {
@@ -1173,39 +1170,6 @@ public class MavibotPartitionBuilder
             e.printStackTrace();
         }
     }
-    
-    public void testPartition() throws Exception
-    {
-        partition.sync();
-        partition.destroy();
-        
-        createPartition();
-        
-        testBTree( masterTableName );
-        
-        SearchRequest req = new SearchRequestImpl();
-        req.setBase( suffixDn );
-        
-        ExprNode filter = new PresenceNode( schemaManager.lookupAttributeTypeRegistry( SchemaConstants.OBJECT_CLASS_AT
) );
-        
-        req.setFilter( filter );
-        req.setScope( SearchScope.SUBTREE );
-        
-        SearchOperationContext searchCtx = new SearchOperationContext( null, req );
-        
-        EntryFilteringCursor cur = partition.search( searchCtx );
-        
-        int count = 0;
-        while( cur.next() )
-        {
-            count++;
-            System.out.println( cur.get() );
-        }
-        
-        cur.close();
-        
-        System.out.println( "Total search results " + count );
-    }
 
     
     /** no qualifier */ int getTotalEntries()
@@ -1226,12 +1190,6 @@ public class MavibotPartitionBuilder
     }
 
 
-    /** no qualifier */ MavibotPartition getPartition()
-    {
-        return partition;
-    }
-
-
     /** no qualifier */ SchemaManager getSchemaManager()
     {
         return schemaManager;
@@ -1244,58 +1202,133 @@ public class MavibotPartitionBuilder
     }
 
 
-    public static void main( String[] args ) throws Exception
+    public static void help()
     {
-        calcLevels( 502, 16 );
+        System.out.println( "Usage" );
+        System.out.println( "java -jar bulkloader.jar <options>" );
+        System.out.println( "Available options are:" );
         
-        File outDir = new File( "/tmp/builder" );
+        Option[] options = Option.values();
         
-        if( outDir.exists() )
+        for( Option o : options )
         {
-            FileUtils.deleteDirectory( outDir );
+            if( o == Option.UNKNOWN )
+            {
+                continue;
+            }
+            
+            System.out.println( o.getText() + "    " + o.getDesc() );
+        }
+    }
+    
+    private static String getArgAt( int position, Option opt, String[] args )
+    {
+        if( position >= args.length )
+        {
+            System.out.println( "No value was provided for the option " + opt.getText() );
+            System.exit( 1 );
         }
         
-        File file = new File( "/Users/dbugger/other-projects/slamd~svn/trunk/slamd/package/slamd/tools/MakeLDIF/100k-users.ldif"
);
+        return args[position];
+    }
+    
+    public static void main( String[] args ) throws Exception
+    {
+        String inFile = null;
+        String outDirPath = null;
+        int numKeysInNode = 16;
+        int rid = 1;
+        boolean cleanOutDir = false;
+        boolean verifyMasterTable = false;
 
-        //file = new File( "/tmp/builder-test.ldif" );
-//        InputStream in = MavibotPartitionBuilder.class.getClassLoader().getResourceAsStream(
"builder-test.ldif" );
-//        FileUtils.copyInputStreamToFile( in, file );
-//        in.close();
-        
-        MavibotPartitionBuilder builder = new MavibotPartitionBuilder( file.getAbsolutePath(),
outDir.getAbsolutePath() );
-        
-        long start = System.currentTimeMillis();
+        if ( args.length < 2 )
+        {
+           help();
+           System.exit( 0 );
+        }
         
-        builder.buildPartition();
+        for( int i =0; i < args.length; i++ )
+        {
+            Option opt = Option.getOpt( args[i] );
+            
+            switch( opt )
+            {
+                case HELP:
+                    help();
+                    System.exit( 0 );
+                    break;
+                    
+                case INPUT_FILE:
+                    inFile = getArgAt( ++i, opt, args );
+                    break;
+
+                case OUT_DIR:
+                    outDirPath = getArgAt( ++i, opt, args );
+                    break;
+
+                case CLEAN_OUT_DIR:
+                    cleanOutDir = true;
+                    break;
+
+                case VERIFY_MASTER_TABLE:
+                    verifyMasterTable = true;
+                    break;
+
+                case NUM_KEYS_PER_NODE:
+                    numKeysInNode = Integer.parseInt( getArgAt( ++i, opt, args ) );
+                    break;
+
+                case DS_RID:
+                    rid = Integer.parseInt( getArgAt( ++i, opt, args ) );
+                    break;
+
+                case UNKNOWN:
+                    System.out.println( "Unknown option " + args[i] );
+                    continue;
+            }
+        }
         
-        long end = System.currentTimeMillis();
+        if( ( inFile == null ) || ( inFile.trim().length() == 0 ) )
+        {
+            System.out.println( "Invalid input file" );
+            return;
+        }
         
-        System.out.println( "Total time taken " + ( end - start ) + "msec" );
+        if( !new File( inFile ).exists() )
+        {
+            System.out.println( "The input file " + inFile + " doesn't exist" );
+            return;
+        }
         
-        //String fwdRdnTree = ApacheSchemaConstants.APACHE_RDN_AT_OID + MavibotRdnIndex.FORWARD_BTREE;
-        builder.testBTree( "master" );
+        //calcLevels( 502, 16 );
         
-        //String revRdnTree = ApacheSchemaConstants.APACHE_RDN_AT_OID + MavibotRdnIndex.REVERSE_BTREE;
-        //builder.testBTree( "2.5.4.0_forward" );
-        //builder.testPartition();
-    }
-    
-    /*
-    public static void main( String[] args ) throws Exception
-    {
-        File outDir = new File( "/tmp/builder" );
+        File outDir = new File( outDirPath );
         
         if( outDir.exists() )
         {
+            if( !cleanOutDir )
+            {
+                System.out.println( "The output directory is not empty, pass " + Option.CLEAN_OUT_DIR.getText()
+ " to force delete the contents or specify a different directory"  );
+                return;
+            }
+            
             FileUtils.deleteDirectory( outDir );
         }
         
+        MavibotPartitionBuilder builder = new MavibotPartitionBuilder( inFile, outDirPath,
numKeysInNode, rid );
         
-        File file = new File( "/tmp/30k.ldif" );
-
-        MavibotPartitionBuilder builder = new MavibotPartitionBuilder( file.getAbsolutePath(),
outDir.getAbsolutePath() );
+        long start = System.currentTimeMillis();
         
         builder.buildPartition();
+        
+        long end = System.currentTimeMillis();
+        
+        System.out.println( "Total time taken " + ( end - start ) + "msec" );
+        
+        if ( verifyMasterTable )
+        {
+            System.out.println( "Verifying the contents of master table" );
+            builder.testBTree( "master" );
+        }
     }
-    */
 }

Added: directory/apacheds/trunk/mavibot-partition/src/main/java/org/apache/directory/mavibot/btree/Option.java
URL: http://svn.apache.org/viewvc/directory/apacheds/trunk/mavibot-partition/src/main/java/org/apache/directory/mavibot/btree/Option.java?rev=1604381&view=auto
==============================================================================
--- directory/apacheds/trunk/mavibot-partition/src/main/java/org/apache/directory/mavibot/btree/Option.java
(added)
+++ directory/apacheds/trunk/mavibot-partition/src/main/java/org/apache/directory/mavibot/btree/Option.java
Sat Jun 21 13:49:57 2014
@@ -0,0 +1,110 @@
+/*
+ *   Licensed to the Apache Software Foundation (ASF) under one
+ *   or more contributor license agreements.  See the NOTICE file
+ *   distributed with this work for additional information
+ *   regarding copyright ownership.  The ASF licenses this file
+ *   to you under the Apache License, Version 2.0 (the
+ *   "License"); you may not use this file except in compliance
+ *   with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   Unless required by applicable law or agreed to in writing,
+ *   software distributed under the License is distributed on an
+ *   "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ *   KIND, either express or implied.  See the License for the
+ *   specific language governing permissions and limitations
+ *   under the License.
+ *
+ */
+package org.apache.directory.mavibot.btree;
+
+
+/**
+ * Command line options for bulk loader
+ * 
+ * @author <a href="mailto:dev@directory.apache.org">Apache Directory Project</a>
+ */
+public enum Option
+{
+    HELP("-h", "Prints the details of the options"),
+    
+    INPUT_FILE("-i", "Path of the LDIF file to be used as input"),
+
+    OUT_DIR("-o", "Path of the directory where the data files will be stored"),
+
+    CLEAN_OUT_DIR("-clean", "Deletes the output directory's contents if present"),
+    
+    NUM_KEYS_PER_NODE("-n", "(optional) The number of keys to be present in each node, default
is 16"),
+
+    DS_RID("-rid", "(optional) The RID value to be used in the entryCSN values, default is
1"),
+
+    VERIFY_MASTER_TABLE("-verify", "(optional) Verifies the master table by just browsing
(entries are not verified)"),
+    
+    UNKNOWN(null, "Unknown Option");
+
+    private String text;
+    private String desc;
+
+
+    private Option( String text, String desc )
+    {
+        this.text = text;
+        this.desc = desc;
+    }
+
+
+    public String getText()
+    {
+        return text;
+    }
+
+
+    public String getDesc()
+    {
+        return desc;
+    }
+
+
+    public static Option getOpt( String opt )
+    {
+        if ( opt == null )
+        {
+            return UNKNOWN;
+        }
+
+        opt = opt.trim();
+
+        if ( opt.equalsIgnoreCase( HELP.text ) )
+        {
+            return HELP;
+        }
+
+        if ( opt.equalsIgnoreCase( VERIFY_MASTER_TABLE.text ) )
+        {
+            return VERIFY_MASTER_TABLE;
+        }
+
+        if ( opt.equalsIgnoreCase( INPUT_FILE.text ) )
+        {
+            return INPUT_FILE;
+        }
+
+        if ( opt.equalsIgnoreCase( OUT_DIR.text ) )
+        {
+            return OUT_DIR;
+        }
+
+        if ( opt.equalsIgnoreCase( CLEAN_OUT_DIR.text ) )
+        {
+            return CLEAN_OUT_DIR;
+        }
+
+        if ( opt.equalsIgnoreCase( NUM_KEYS_PER_NODE.text ) )
+        {
+            return NUM_KEYS_PER_NODE;
+        }
+
+        return UNKNOWN;
+    }
+}



Mime
View raw message