directory-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From seelm...@apache.org
Subject svn commit: r906716 - in /directory/sandbox/seelmann/hbase-partition: ./ src/main/java/org/apache/directory/server/core/partition/hbase/ src/main/java/org/apache/directory/server/core/partition/hbase/table/ src/test/java/org/apache/directory/server/cor...
Date Thu, 04 Feb 2010 23:03:49 GMT
Author: seelmann
Date: Thu Feb  4 23:03:48 2010
New Revision: 906716

URL: http://svn.apache.org/viewvc?rev=906716&view=rev
Log:
Fixed embedded HBase partition

Modified:
    directory/sandbox/seelmann/hbase-partition/pom.xml
    directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/Cache.java
    directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/EmbeddedHBase.java
    directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/HBaseStore.java
    directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/table/HBaseMasterTable.java
    directory/sandbox/seelmann/hbase-partition/src/test/java/org/apache/directory/server/core/partition/hbase/HBaseClusterTestCaseAdapter.java
    directory/sandbox/seelmann/hbase-partition/src/test/java/org/apache/directory/server/core/partition/hbase/HBaseEmbeddedRunner.java

Modified: directory/sandbox/seelmann/hbase-partition/pom.xml
URL: http://svn.apache.org/viewvc/directory/sandbox/seelmann/hbase-partition/pom.xml?rev=906716&r1=906715&r2=906716&view=diff
==============================================================================
--- directory/sandbox/seelmann/hbase-partition/pom.xml (original)
+++ directory/sandbox/seelmann/hbase-partition/pom.xml Thu Feb  4 23:03:48 2010
@@ -97,38 +97,32 @@
       <groupId>org.apache.directory.hbase</groupId>
       <artifactId>hadoop-test</artifactId>
       <version>0.20.1</version>
-      <scope>test</scope>
     </dependency>
     <dependency>
       <groupId>org.apache.directory.hbase</groupId>
       <artifactId>hbase-test</artifactId>
       <version>0.20.3-RC3</version>
-      <scope>test</scope>
     </dependency>
     <dependency>
       <groupId>org.apache.directory.hbase</groupId>
       <artifactId>zookeeper</artifactId>
       <version>3.2.2</version>
-      <scope>test</scope>
     </dependency>
     <!-- HBase transitive test depencencies -->
     <dependency>
       <groupId>commons-lang</groupId>
       <artifactId>commons-lang</artifactId>
       <version>2.4</version>
-      <scope>test</scope>
     </dependency>
     <dependency>
       <groupId>commons-httpclient</groupId>
       <artifactId>commons-httpclient</artifactId>
       <version>3.0.1</version>
-      <scope>test</scope>
     </dependency>
     <dependency>
       <groupId>commons-cli</groupId>
       <artifactId>commons-cli</artifactId>
       <version>1.2</version>
-      <scope>test</scope>
     </dependency>
     <dependency>
       <groupId>org.mortbay.jetty</groupId>

Modified: directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/Cache.java
URL: http://svn.apache.org/viewvc/directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/Cache.java?rev=906716&r1=906715&r2=906716&view=diff
==============================================================================
--- directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/Cache.java
(original)
+++ directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/Cache.java
Thu Feb  4 23:03:48 2010
@@ -67,7 +67,10 @@
     public Cache( int size, int ttl )
     {
         this.ttl = ttl;
-        this.map = new LRUMap( size );
+        if ( size > 0 )
+        {
+            this.map = new LRUMap( size );
+        }
     }
 
 
@@ -79,6 +82,11 @@
      */
     public synchronized void put( K key, V value )
     {
+        if ( map == null )
+        {
+            return;
+        }
+
         if ( value == null )
         {
             map.remove( key );
@@ -99,6 +107,11 @@
      */
     public synchronized boolean contains( K key )
     {
+        if ( map == null )
+        {
+            return false;
+        }
+
         return get( key ) != null;
     }
 
@@ -112,6 +125,11 @@
      */
     public synchronized V get( K key )
     {
+        if ( map == null )
+        {
+            return null;
+        }
+
         Entry entry = ( Entry ) map.get( key );
         if ( entry != null )
         {
@@ -137,6 +155,11 @@
      */
     public synchronized void clear()
     {
+        if ( map == null )
+        {
+            return;
+        }
+
         map.clear();
     }
 

Modified: directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/EmbeddedHBase.java
URL: http://svn.apache.org/viewvc/directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/EmbeddedHBase.java?rev=906716&r1=906715&r2=906716&view=diff
==============================================================================
--- directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/EmbeddedHBase.java
(original)
+++ directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/EmbeddedHBase.java
Thu Feb  4 23:03:48 2010
@@ -26,8 +26,11 @@
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.MiniHBaseCluster;
 import org.apache.hadoop.hbase.MiniZooKeeperCluster;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
 import org.apache.hadoop.hbase.client.HConnectionManager;
 import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.util.Bytes;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -45,67 +48,131 @@
 
     private static final Logger LOG = LoggerFactory.getLogger( EmbeddedHBase.class );
 
-    protected File workingDirectory;
-    protected HBaseConfiguration conf;
-    protected MiniHBaseCluster cluster;
-    protected MiniZooKeeperCluster zooKeeperCluster;
+    private static final EmbeddedHBase INSTANCE = new EmbeddedHBase();
+
+    private File workingDirectory;
+    private HBaseConfiguration conf;
+    private MiniHBaseCluster cluster;
+    private MiniZooKeeperCluster zooKeeperCluster;
+    private int count = 0;
+
+
+    private EmbeddedHBase()
+    {
+    }
 
 
     /**
-     * Creates a new instance of EmbeddedHBase.
-     * Note that the configuration is modified to use the local file system
-     * rather than an DFS.
+     * Gets a EmbeddedHBase instance.
      *
      * @param workingDirectory the working directory
-     * @param conf the HBase configuration
      * @throws Exception
      */
-    public EmbeddedHBase( File workingDirectory, HBaseConfiguration conf ) throws Exception
+    public static EmbeddedHBase getInstance( File workingDirectory )
     {
-        this.workingDirectory = workingDirectory;
-        this.conf = conf;
-
-        // use local filesystem
-        conf.set( "fs.default.name", "file:///" );
-        // zookeeper working directory
-        conf.set( "test.build.data", new File( workingDirectory, "zookeeper" ).getAbsolutePath()
);
-        // hbase working directory
-        conf.set( HConstants.HBASE_DIR, new File( workingDirectory, "hbase" ).toURI().toURL().toString()
);
+        synchronized ( INSTANCE )
+        {
+            if ( INSTANCE.workingDirectory == null )
+            {
+                INSTANCE.workingDirectory = workingDirectory;
+                INSTANCE.conf = new HBaseConfiguration();
+            }
+            return INSTANCE;
+        }
     }
 
 
     /**
      * Starts the embedded HBase cluster.
      * 
+     * @param tableName the table name of the table to wait for to become ready.
+     * 
      * @throws Exception the exception
      */
-    public void start() throws Exception
+    public synchronized void start( String tableName ) throws Exception
     {
-        try
+        if ( count == 0 )
         {
-            // Note that this is done before we create the MiniHBaseCluster because we
-            // need to edit the config to add the ZooKeeper servers.
-            zooKeeperCluster = new MiniZooKeeperCluster();
-            int clientPort = this.zooKeeperCluster.startup( workingDirectory );
-            conf.set( "hbase.zookeeper.property.clientPort", Integer.toString( clientPort
) );
+            LOG.info( "Start embedded HBase" );
+            try
+            {
+                // hdfs working directory
+                System.setProperty( "test.build.data", workingDirectory.getAbsolutePath()
);
+                // zookeeper working directory
+                conf.set( "test.build.data", new File( workingDirectory, "zookeeper" ).getAbsolutePath()
);
+                // use local filesystem
+                conf.set( "fs.default.name", "file:///" );
+                // hbase working directory
+                conf.set( HConstants.HBASE_DIR, new File( workingDirectory, "hbase" ).toURI().toURL().toString()
);
+
+                // Note that this is done before we create the MiniHBaseCluster because we
+                // need to edit the config to add the ZooKeeper servers.
+                zooKeeperCluster = new MiniZooKeeperCluster();
+                int clientPort = this.zooKeeperCluster.startup( workingDirectory );
+                conf.set( "hbase.zookeeper.property.clientPort", Integer.toString( clientPort
) );
 
-            // start the mini cluster
-            cluster = new MiniHBaseCluster( conf, 1 );
+                // start the mini cluster
+                cluster = new MiniHBaseCluster( conf, 1 );
+            }
+            catch ( Exception e )
+            {
+                LOG.error( "Error while starting embedded HBase.", e );
+                if ( cluster != null )
+                {
+                    cluster.shutdown();
+                }
+                if ( zooKeeperCluster != null )
+                {
+                    zooKeeperCluster.shutdown();
+                }
+                throw e;
+            }
 
         }
-        catch ( Exception e )
+
+        // increment counter
+        count++;
+
+        /*
+         *  Wait till the tree table is ready.
+         */
+        LOG.debug( "Waiting for table {} to become ready...", tableName );
+
+        // remember the current retry settings
+        String retriesNumber = conf.get( "hbase.client.retries.number" );
+        String pause = conf.get( "hbase.client.pause" );
+        // set some extended retry settings, 2 minutes should be enough...
+        conf.set( "hbase.client.retries.number", "60" ); // number of retries
+        conf.set( "hbase.client.pause", "2000" ); // pause in milli seconds
+        try
         {
-            LOG.error( "Exception in start.", e );
-            if ( cluster != null )
+            HBaseAdmin admin = new HBaseAdmin( conf );
+
+            if ( tableName != null && admin.tableExists( tableName ) )
             {
-                cluster.shutdown();
+                HTable table = new HTable( conf, tableName );
+                Get get = new Get( Bytes.toBytes( 0L ) );
+                table.get( get );
+                table.close();
+                LOG.debug( "Table {} is ready.", tableName );
             }
-            if ( zooKeeperCluster != null )
+            else
             {
-                zooKeeperCluster.shutdown();
+                LOG.debug( "Table {} doesn't exist, seems to be a clean start.", tableName
);
             }
+        }
+        catch ( Exception e )
+        {
+            e.printStackTrace();
+            LOG.error( "Error while waiting for table to become ready.", e );
             throw e;
         }
+        finally
+        {
+            // reset the retry settings
+            conf.set( "hbase.client.retries.number", retriesNumber );
+            conf.set( "hbase.client.pause", pause );
+        }
     }
 
 
@@ -114,18 +181,33 @@
      * 
      * @throws Exception the exception
      */
-    public void shutdown() throws Exception
+    public synchronized void shutdown() throws Exception
     {
+        if ( count > 1 )
+        {
+            // dont't stop the cluster till the last shutdown.
+            count--;
+            return;
+        }
+
         try
         {
+            LOG.info( "Shutdown embedded HBase" );
+            count = 0;
+
             // open the META table now to ensure cluster is running before shutdown.
             new HTable( conf, HConstants.META_TABLE_NAME );
 
             HConnectionManager.deleteConnectionInfo( conf, true );
             if ( this.cluster != null )
             {
+                LOG.debug( "Shutdown cluster..." );
                 this.cluster.shutdown();
+                LOG.debug( "...done" );
+
+                LOG.debug( "Shutdown zookeeper..." );
                 this.zooKeeperCluster.shutdown();
+                LOG.debug( "...done" );
             }
         }
         catch ( Exception e )
@@ -134,4 +216,15 @@
         }
     }
 
+
+    /**
+     * Gets the HBase configuration.
+     * 
+     * @return the HBase configuration
+     */
+    public HBaseConfiguration getConfiguration()
+    {
+        return conf;
+    }
+
 }

Modified: directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/HBaseStore.java
URL: http://svn.apache.org/viewvc/directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/HBaseStore.java?rev=906716&r1=906715&r2=906716&view=diff
==============================================================================
--- directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/HBaseStore.java
(original)
+++ directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/HBaseStore.java
Thu Feb  4 23:03:48 2010
@@ -92,12 +92,15 @@
 
     public void init( SchemaManager schemaManager ) throws Exception
     {
-        this.configuration = new HBaseConfiguration();
         if ( workingDirectory != null )
         {
-            // note that the configuration is modified to use the local file system
-            this.embeddedHBase = new EmbeddedHBase( workingDirectory, configuration );
-            this.embeddedHBase.start();
+            embeddedHBase = EmbeddedHBase.getInstance( workingDirectory );
+            embeddedHBase.start( tablePrefix + HBaseMasterTable.TREE_TABLE );
+            configuration = embeddedHBase.getConfiguration();
+        }
+        else
+        {
+            configuration = new HBaseConfiguration();
         }
 
         this.schemaManager = schemaManager;
@@ -139,11 +142,6 @@
 
     public void destroy() throws Exception
     {
-        if ( embeddedHBase != null )
-        {
-            embeddedHBase.shutdown();
-            embeddedHBase = null;
-        }
         if ( masterTable != null )
         {
             masterTable.close();
@@ -164,6 +162,12 @@
         oneAliasIndex.close();
         subAliasIndex.close();
 
+        if ( embeddedHBase != null )
+        {
+            embeddedHBase.shutdown();
+            embeddedHBase = null;
+        }
+
         schemaManager = null;
     }
 

Modified: directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/table/HBaseMasterTable.java
URL: http://svn.apache.org/viewvc/directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/table/HBaseMasterTable.java?rev=906716&r1=906715&r2=906716&view=diff
==============================================================================
--- directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/table/HBaseMasterTable.java
(original)
+++ directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/table/HBaseMasterTable.java
Thu Feb  4 23:03:48 2010
@@ -84,8 +84,8 @@
     public static final byte[] NORM_ATTRIBUTE_DELIMITER_BYTES = new byte[]
         { NORM_ATTRIBUTE_DELIMITER };
 
-    private static final String MASTER_TABLE = "master";
-    private static final String TREE_TABLE = "tree";
+    public static final String MASTER_TABLE = "master";
+    public static final String TREE_TABLE = "tree";
 
     private HBaseConfiguration configuration;
     private SchemaManager schemaManager;

Modified: directory/sandbox/seelmann/hbase-partition/src/test/java/org/apache/directory/server/core/partition/hbase/HBaseClusterTestCaseAdapter.java
URL: http://svn.apache.org/viewvc/directory/sandbox/seelmann/hbase-partition/src/test/java/org/apache/directory/server/core/partition/hbase/HBaseClusterTestCaseAdapter.java?rev=906716&r1=906715&r2=906716&view=diff
==============================================================================
--- directory/sandbox/seelmann/hbase-partition/src/test/java/org/apache/directory/server/core/partition/hbase/HBaseClusterTestCaseAdapter.java
(original)
+++ directory/sandbox/seelmann/hbase-partition/src/test/java/org/apache/directory/server/core/partition/hbase/HBaseClusterTestCaseAdapter.java
Thu Feb  4 23:03:48 2010
@@ -51,9 +51,12 @@
         super( 1, startDfs );
         setName( clazz.getName() );
 
-        // use target as test  directory base
+        // use target as test  directory base...
         File testDir = new File( "target/data" );
+        // ...for zookeeper
         conf.set( TEST_DIRECTORY_KEY, testDir.getAbsolutePath() );
+        // ...for hdfs
+        System.setProperty( "test.build.data", testDir.getAbsolutePath() );
 
         // setup local file system if no DFS is used
         if ( !startDfs )

Modified: directory/sandbox/seelmann/hbase-partition/src/test/java/org/apache/directory/server/core/partition/hbase/HBaseEmbeddedRunner.java
URL: http://svn.apache.org/viewvc/directory/sandbox/seelmann/hbase-partition/src/test/java/org/apache/directory/server/core/partition/hbase/HBaseEmbeddedRunner.java?rev=906716&r1=906715&r2=906716&view=diff
==============================================================================
--- directory/sandbox/seelmann/hbase-partition/src/test/java/org/apache/directory/server/core/partition/hbase/HBaseEmbeddedRunner.java
(original)
+++ directory/sandbox/seelmann/hbase-partition/src/test/java/org/apache/directory/server/core/partition/hbase/HBaseEmbeddedRunner.java
Thu Feb  4 23:03:48 2010
@@ -19,8 +19,6 @@
  */
 package org.apache.directory.server.core.partition.hbase;
 
-import java.util.Set;
-
 import org.apache.directory.server.annotations.CreateLdapServer;
 import org.apache.directory.server.annotations.CreateTransport;
 import org.apache.directory.server.core.CoreSession;
@@ -29,7 +27,6 @@
 import org.apache.directory.server.core.annotations.CreatePartition;
 import org.apache.directory.server.core.integ.AbstractLdapTestUnit;
 import org.apache.directory.server.core.integ.FrameworkRunner;
-import org.apache.directory.server.core.partition.Partition;
 import org.apache.directory.server.core.partition.hbase.index.HBaseUserColumnIndex;
 import org.apache.directory.server.core.partition.hbase.index.HBaseUserRowIndex;
 import org.apache.directory.server.core.partition.hbase.it.AbstractHBasePartitionIT;
@@ -41,8 +38,8 @@
 
 
 /**
- * Starts up an ApacheDS LDAP server with an HBase partition.
- * The partition starts up an embedded HBase partition.  
+ * Starts up an ApacheDS LDAP server with two HBase partitions.
+ * The partitions starts up one embedded HBase partition.  
  * 
  * @author <a href="mailto:dev@directory.apache.org">Apache Directory Project</a>
  * @version $Rev$, $Date$
@@ -59,22 +56,22 @@
             type = HBaseEmbeddedPartition.class,
             cacheSize=1000,
             indexes = {
-                @CreateIndex( attribute="cn", cacheSize=100, type = HBaseUserColumnIndex.class
),
-                @CreateIndex( attribute="uid", cacheSize=100, type = HBaseUserColumnIndex.class
),
+                @CreateIndex( attribute="cn", cacheSize=1000, type = HBaseUserColumnIndex.class
),
+                @CreateIndex( attribute="uid", cacheSize=1000, type = HBaseUserColumnIndex.class
),
                 
                 @CreateIndex( attribute="dc", cacheSize=10, type = HBaseUserRowIndex.class
),
                 @CreateIndex( attribute="o", cacheSize=10, type = HBaseUserRowIndex.class
),
-                @CreateIndex( attribute="ou", cacheSize=100, type = HBaseUserRowIndex.class
),
+                @CreateIndex( attribute="ou", cacheSize=10, type = HBaseUserRowIndex.class
),
 
-                @CreateIndex( attribute="objectClass", cacheSize=100, type = HBaseUserRowIndex.class
),
+                @CreateIndex( attribute="objectClass", cacheSize=100, type = HBaseUserRowIndex.class
)
             }
         )
-//        ,
-//        @CreatePartition(
-//            name = "sevenSeas", 
-//            suffix = "o=sevenSeas", 
-//            type = HBaseEmbeddedPartition.class
-//        ) 
+        ,
+        @CreatePartition(
+            name = "sevenSeas", 
+            suffix = "o=sevenSeas", 
+            type = HBaseEmbeddedPartition.class
+        ) 
     })
 @CreateLdapServer(transports =
     { @CreateTransport(protocol = "LDAP", port = 10389, nbThreads=16) })
@@ -88,8 +85,6 @@
     public void initTestData() throws Exception
     {
         session = ldapServer.getDirectoryService().getAdminSession();
-        Set<? extends Partition> partitions = ldapServer.getDirectoryService().getPartitions();
-        System.out.println(partitions);
 
         if ( !session.exists( new LdapDN( "o=hbase" ) ) || !session.exists( new LdapDN( "ou=test-ou,o=hbase"
) )
             || !session.exists( new LdapDN( "cn=test-person,ou=test-ou,o=hbase" ) ) )



Mime
View raw message