directory-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kayyag...@apache.org
Subject svn commit: r1547297 - in /directory/apacheds/trunk/mavibot-partition/src: main/java/org/apache/directory/server/core/partition/impl/btree/mavibot/ test/java/org/apache/directory/server/core/partition/impl/btree/mavibot/
Date Tue, 03 Dec 2013 06:17:26 GMT
Author: kayyagari
Date: Tue Dec  3 06:17:25 2013
New Revision: 1547297

URL: http://svn.apache.org/r1547297
Log:
intiial commit related to bulk loading data into a mavibot partition

Added:
    directory/apacheds/trunk/mavibot-partition/src/main/java/org/apache/directory/server/core/partition/impl/btree/mavibot/LdifTupleComparator.java
    directory/apacheds/trunk/mavibot-partition/src/main/java/org/apache/directory/server/core/partition/impl/btree/mavibot/LdifTupleReaderWriter.java
    directory/apacheds/trunk/mavibot-partition/src/test/java/org/apache/directory/server/core/partition/impl/btree/mavibot/LdifBulkLoaderTest.java

Added: directory/apacheds/trunk/mavibot-partition/src/main/java/org/apache/directory/server/core/partition/impl/btree/mavibot/LdifTupleComparator.java
URL: http://svn.apache.org/viewvc/directory/apacheds/trunk/mavibot-partition/src/main/java/org/apache/directory/server/core/partition/impl/btree/mavibot/LdifTupleComparator.java?rev=1547297&view=auto
==============================================================================
--- directory/apacheds/trunk/mavibot-partition/src/main/java/org/apache/directory/server/core/partition/impl/btree/mavibot/LdifTupleComparator.java
(added)
+++ directory/apacheds/trunk/mavibot-partition/src/main/java/org/apache/directory/server/core/partition/impl/btree/mavibot/LdifTupleComparator.java
Tue Dec  3 06:17:25 2013
@@ -0,0 +1,57 @@
+/*
+ *   Licensed to the Apache Software Foundation (ASF) under one
+ *   or more contributor license agreements.  See the NOTICE file
+ *   distributed with this work for additional information
+ *   regarding copyright ownership.  The ASF licenses this file
+ *   to you under the Apache License, Version 2.0 (the
+ *   "License"); you may not use this file except in compliance
+ *   with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   Unless required by applicable law or agreed to in writing,
+ *   software distributed under the License is distributed on an
+ *   "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ *   KIND, either express or implied.  See the License for the
+ *   specific language governing permissions and limitations
+ *   under the License.
+ *
+ */
+package org.apache.directory.server.core.partition.impl.btree.mavibot;
+
+import java.util.Comparator;
+
+import org.apache.directory.api.ldap.model.name.Dn;
+import org.apache.directory.mavibot.btree.Tuple;
+
+/**
+ * TODO LdifTupleComparator.
+ *
+ * @author <a href="mailto:dev@directory.apache.org">Apache Directory Project</a>
+ */
+public class LdifTupleComparator implements Comparator<Tuple<Dn, String>>
+{
+
+    @Override
+    public int compare( Tuple<Dn, String> t1, Tuple<Dn, String> t2 )
+    {
+        Dn dn1 = t1.getKey();
+        Dn dn2 = t2.getKey();
+        
+        if( dn1.isAncestorOf( dn2 ) )
+        {
+            return -1;
+        }
+        else if( dn2.isAncestorOf( dn1 ) )
+        {
+            return 1;
+        }
+        else if ( dn1.equals( dn2 ) )
+        {
+            return 0;
+        }
+        
+        return dn1.getNormName().compareTo( dn2.getNormName() );
+    }
+
+}

Added: directory/apacheds/trunk/mavibot-partition/src/main/java/org/apache/directory/server/core/partition/impl/btree/mavibot/LdifTupleReaderWriter.java
URL: http://svn.apache.org/viewvc/directory/apacheds/trunk/mavibot-partition/src/main/java/org/apache/directory/server/core/partition/impl/btree/mavibot/LdifTupleReaderWriter.java?rev=1547297&view=auto
==============================================================================
--- directory/apacheds/trunk/mavibot-partition/src/main/java/org/apache/directory/server/core/partition/impl/btree/mavibot/LdifTupleReaderWriter.java
(added)
+++ directory/apacheds/trunk/mavibot-partition/src/main/java/org/apache/directory/server/core/partition/impl/btree/mavibot/LdifTupleReaderWriter.java
Tue Dec  3 06:17:25 2013
@@ -0,0 +1,119 @@
+/*
+ *   Licensed to the Apache Software Foundation (ASF) under one
+ *   or more contributor license agreements.  See the NOTICE file
+ *   distributed with this work for additional information
+ *   regarding copyright ownership.  The ASF licenses this file
+ *   to you under the Apache License, Version 2.0 (the
+ *   "License"); you may not use this file except in compliance
+ *   with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   Unless required by applicable law or agreed to in writing,
+ *   software distributed under the License is distributed on an
+ *   "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ *   KIND, either express or implied.  See the License for the
+ *   specific language governing permissions and limitations
+ *   under the License.
+ *
+ */
+package org.apache.directory.server.core.partition.impl.btree.mavibot;
+
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+
+import org.apache.directory.api.ldap.model.ldif.LdifEntry;
+import org.apache.directory.api.ldap.model.ldif.LdifReader;
+import org.apache.directory.api.ldap.model.name.Dn;
+import org.apache.directory.mavibot.btree.Tuple;
+import org.apache.directory.mavibot.btree.util.TupleReaderWriter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+/**
+ * TODO LdifTupleReaderWriter.
+ *
+ * @author <a href="mailto:dev@directory.apache.org">Apache Directory Project</a>
+ */
+public class LdifTupleReaderWriter implements TupleReaderWriter<Dn, String>
+{
+
+    private LdifReader reader = null;
+
+    private static final Logger LOG = LoggerFactory.getLogger( LdifTupleReaderWriter.class
);
+
+
+    @Override
+    public Tuple<Dn, String> readSortedTuple( DataInputStream in )
+    {
+        try
+        {
+            if( in.available() > 0 )
+            {
+                Tuple<Dn, String> t = new Tuple<Dn, String>();
+                t.setKey( new Dn( in.readUTF() ) );
+                t.setValue( in.readUTF() );
+                
+                return t;
+            }
+        }
+        catch ( Exception e )
+        {
+            LOG.error( "Failed to read a sorted tuple", e );
+        }
+        
+        return null;
+    }
+
+
+    @Override
+    public Tuple<Dn, String> readUnsortedTuple( DataInputStream in )
+    {
+        try
+        {
+            if ( reader == null )
+            {
+                reader = new LdifReader( in );
+            }
+        }
+        catch ( Exception e )
+        {
+            String msg = "Failed to open the LDIF input stream";
+            LOG.error( msg, e );
+
+            throw new RuntimeException( msg, e );
+        }
+
+        Tuple<Dn, String> t = null;
+
+        if ( reader.hasNext() )
+        {
+            LdifEntry ldifEntry = reader.next();
+
+            if ( ldifEntry == null )
+            {
+                throw new IllegalStateException(
+                    "Received null entry while parsing, check the LDIF file for possible
incorrect/corrupted entries" );
+            }
+
+            t = new Tuple<Dn, String>();
+
+            t.setKey( ldifEntry.getDn() );
+            t.setValue( ldifEntry.getOffset() + ":" + ldifEntry.getLengthBeforeParsing()
);
+        }
+
+        return t;
+    }
+
+
+    @Override
+    public void storeSortedTuple( Tuple<Dn, String> t, DataOutputStream out ) throws
IOException
+    {
+        out.writeUTF( t.getKey().getName() );
+        out.writeUTF( t.getValue() );
+    }
+
+}

Added: directory/apacheds/trunk/mavibot-partition/src/test/java/org/apache/directory/server/core/partition/impl/btree/mavibot/LdifBulkLoaderTest.java
URL: http://svn.apache.org/viewvc/directory/apacheds/trunk/mavibot-partition/src/test/java/org/apache/directory/server/core/partition/impl/btree/mavibot/LdifBulkLoaderTest.java?rev=1547297&view=auto
==============================================================================
--- directory/apacheds/trunk/mavibot-partition/src/test/java/org/apache/directory/server/core/partition/impl/btree/mavibot/LdifBulkLoaderTest.java
(added)
+++ directory/apacheds/trunk/mavibot-partition/src/test/java/org/apache/directory/server/core/partition/impl/btree/mavibot/LdifBulkLoaderTest.java
Tue Dec  3 06:17:25 2013
@@ -0,0 +1,118 @@
+/*
+ *   Licensed to the Apache Software Foundation (ASF) under one
+ *   or more contributor license agreements.  See the NOTICE file
+ *   distributed with this work for additional information
+ *   regarding copyright ownership.  The ASF licenses this file
+ *   to you under the Apache License, Version 2.0 (the
+ *   "License"); you may not use this file except in compliance
+ *   with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   Unless required by applicable law or agreed to in writing,
+ *   software distributed under the License is distributed on an
+ *   "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ *   KIND, either express or implied.  See the License for the
+ *   specific language governing permissions and limitations
+ *   under the License.
+ *
+ */
+package org.apache.directory.server.core.partition.impl.btree.mavibot;
+
+import java.io.File;
+import java.io.FileWriter;
+import java.util.Iterator;
+
+import org.apache.directory.api.ldap.model.name.Dn;
+import org.apache.directory.api.ldap.model.name.Rdn;
+import org.apache.directory.mavibot.btree.Tuple;
+import org.apache.directory.mavibot.btree.managed.BulkDataSorter;
+import org.junit.Ignore;
+
+/**
+ * TODO LdifBulkLoaderTest.
+ *
+ * @author <a href="mailto:dev@directory.apache.org">Apache Directory Project</a>
+ */
+@Ignore
+public class LdifBulkLoaderTest
+{
+    String personTemplate =  "# just a comment\n"+
+                    "dn: uid={uid},{parent}\n" +
+                    "objectClass: top\n" +
+                    "objectClass: person\n" +
+                    "objectClass: organizationalPerson\n" +
+                    "objectClass: inetOrgPerson\n" +
+                    "givenName: {uid}_{uid}\n" +
+                    "sn: {uid}_sn\n" +
+                    "cn: {uid}_cn\n" +
+                    "uid: {uid}\n\n";
+
+    String ouTemplate = "# just another comment\n"+
+                        "dn: {ouDn}\n" +
+                        "objectclass: top\n" +
+                        "objectclass: organizationalUnit\n" +
+                        "ou: {ou}\n\n";
+    
+    private void sort() throws Exception
+    {
+        LdifTupleReaderWriter readerWriter = new LdifTupleReaderWriter();
+        LdifTupleComparator tupleComparator = new LdifTupleComparator();
+        
+        BulkDataSorter bs = new BulkDataSorter<Dn, String>( readerWriter, tupleComparator,
2 );
+        
+        File file = createLdif();
+        
+        bs.sort( file );
+        
+        Iterator<Tuple<Dn, String>> itr = bs.getMergeSortedTuples();
+        while( itr.hasNext() )
+        {
+            Tuple t = itr.next();
+            System.out.println( t.getKey() + " - " + t.getValue() );
+        }
+        
+        //BTreeBuilder<, V>
+    }
+    
+    private File createLdif() throws Exception
+    {
+        File file = File.createTempFile( "bulkload-test", ".ldif" );
+        file.deleteOnExit();
+        
+        FileWriter fw = new FileWriter( file );
+        
+        Dn parentDn = new Dn( "ou=grandchildren,ou=children,ou=parent,ou=system" );
+        
+        Dn currentDn = parentDn;
+        
+        for( Rdn rdn : parentDn.getRdns() )
+        {
+            for( int i =0; i< 2; i++ )
+            {
+                String user = personTemplate.replace( "{uid}", "user"+i );
+                user = user.replace( "{parent}", currentDn.getName() );
+
+                fw.write( user );
+            }
+            
+            
+            String userBranch = ouTemplate.replace( "{ou}", rdn.getValue().getString() );
+            userBranch = userBranch.replace( "{ouDn}", currentDn.getName() );
+            
+            fw.write( userBranch );
+            
+            currentDn = currentDn.getParent();
+        }
+        
+        fw.close();
+
+        return file;
+    }
+    
+    public static void main( String[] args ) throws Exception
+    {
+        LdifBulkLoaderTest bl = new LdifBulkLoaderTest();
+        bl.sort();
+    }
+}



Mime
View raw message