Return-Path: Delivered-To: apmail-directory-commits-archive@www.apache.org Received: (qmail 20098 invoked from network); 1 Feb 2010 07:09:00 -0000 Received: from hermes.apache.org (HELO mail.apache.org) (140.211.11.3) by minotaur.apache.org with SMTP; 1 Feb 2010 07:09:00 -0000 Received: (qmail 8660 invoked by uid 500); 1 Feb 2010 07:09:00 -0000 Delivered-To: apmail-directory-commits-archive@directory.apache.org Received: (qmail 8598 invoked by uid 500); 1 Feb 2010 07:09:00 -0000 Mailing-List: contact commits-help@directory.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@directory.apache.org Delivered-To: mailing list commits@directory.apache.org Received: (qmail 8589 invoked by uid 99); 1 Feb 2010 07:09:00 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 01 Feb 2010 07:09:00 +0000 X-ASF-Spam-Status: No, hits=-1999.6 required=10.0 tests=ALL_TRUSTED,SUBJECT_FUZZY_TION X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 01 Feb 2010 07:08:47 +0000 Received: by eris.apache.org (Postfix, from userid 65534) id 0C67723889E2; Mon, 1 Feb 2010 07:08:25 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit Subject: svn commit: r905194 [1/2] - in /directory/sandbox/seelmann/hbase-partition/src: main/java/org/apache/directory/server/core/partition/hbase/ main/java/org/apache/directory/server/core/partition/hbase/cursor/ main/java/org/apache/directory/server/core/pa... Date: Mon, 01 Feb 2010 07:08:24 -0000 To: commits@directory.apache.org From: seelmann@apache.org X-Mailer: svnmailer-1.0.8 Message-Id: <20100201070825.0C67723889E2@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: seelmann Date: Mon Feb 1 07:08:23 2010 New Revision: 905194 URL: http://svn.apache.org/viewvc?rev=905194&view=rev Log: o added support for embedded hbase cluster o manage index tables within index classes o configurable indices Added: directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/EmbeddedHBase.java directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/HBaseDistributedPartition.java - copied, changed from r902620, directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/HBasePartition.java directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/HBaseEmbeddedPartition.java directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/index/HBaseSubAliasIndex.java directory/sandbox/seelmann/hbase-partition/src/test/java/org/apache/directory/server/core/partition/hbase/HBaseDistributedRunner.java - copied, changed from r902620, directory/sandbox/seelmann/hbase-partition/src/test/java/org/apache/directory/server/core/partition/hbase/HBaseRunner.java directory/sandbox/seelmann/hbase-partition/src/test/java/org/apache/directory/server/core/partition/hbase/HBaseEmbeddedRunner.java Removed: directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/HBasePartition.java directory/sandbox/seelmann/hbase-partition/src/test/java/org/apache/directory/server/core/partition/hbase/HBaseRunner.java Modified: directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/Cache.java directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/HBaseStore.java directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/cursor/HBasePresenceIndexCursor.java directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/index/AbstractHBaseIndex.java directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/index/HBaseNdnIndex.java directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/index/HBaseOneLevelIndex.java directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/index/HBasePresenceIndex.java directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/index/HBaseSubLevelIndex.java directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/index/HBaseUserColumnIndex.java directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/index/HBaseUserIndex.java directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/index/HBaseUserRowIndex.java directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/table/HBaseColumnIndexTable.java directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/table/HBaseIndexTable.java directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/table/HBaseIndexTableBase.java directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/table/HBaseMasterTable.java directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/table/HBasePresenceIndexTable.java directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/table/HBaseRowIndexTable.java directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/table/HBaseTableHelper.java directory/sandbox/seelmann/hbase-partition/src/test/java/org/apache/directory/server/core/partition/hbase/it/AbstractHBasePartitionIT.java directory/sandbox/seelmann/hbase-partition/src/test/java/org/apache/directory/server/core/partition/hbase/it/HBasePartitionIT.java directory/sandbox/seelmann/hbase-partition/src/test/java/org/apache/directory/server/core/partition/hbase/it/HBasePartitionPerformanceIT.java directory/sandbox/seelmann/hbase-partition/src/test/java/org/apache/directory/server/core/partition/hbase/it/HBaseRunner.java directory/sandbox/seelmann/hbase-partition/src/test/java/org/apache/directory/server/core/partition/hbase/table/HBaseIndexTableTest.java directory/sandbox/seelmann/hbase-partition/src/test/java/org/apache/directory/server/core/partition/hbase/table/HBaseMasterTableTest.java directory/sandbox/seelmann/hbase-partition/src/test/java/org/apache/directory/server/core/partition/hbase/table/HBasePresenceIndexTableTest.java Modified: directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/Cache.java URL: http://svn.apache.org/viewvc/directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/Cache.java?rev=905194&r1=905193&r2=905194&view=diff ============================================================================== --- directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/Cache.java (original) +++ directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/Cache.java Mon Feb 1 07:08:23 2010 @@ -31,6 +31,9 @@ */ public class Cache { + private static final int DEFAULT_SIZE = 100; + private static final int DEFAULT_TTL = 60 * 1000; + private int ttl; private LRUMap map; @@ -40,7 +43,18 @@ */ public Cache() { - this( 100, 60 * 1000 ); + this( DEFAULT_SIZE ); + } + + + /** + * Creates a new instance of Cache with an TTL of 60 seconds.. + * + * @param size the cache size + */ + public Cache( int size ) + { + this( size, DEFAULT_TTL ); } Added: directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/EmbeddedHBase.java URL: http://svn.apache.org/viewvc/directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/EmbeddedHBase.java?rev=905194&view=auto ============================================================================== --- directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/EmbeddedHBase.java (added) +++ directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/EmbeddedHBase.java Mon Feb 1 07:08:23 2010 @@ -0,0 +1,137 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +package org.apache.directory.server.core.partition.hbase; + + +import java.io.File; + +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.MiniHBaseCluster; +import org.apache.hadoop.hbase.MiniZooKeeperCluster; +import org.apache.hadoop.hbase.client.HConnectionManager; +import org.apache.hadoop.hbase.client.HTable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + + +/** + * An embedded HBase cluster. Starts HBase and Zookeeper in-process. + * Uses the local file system (no distributed file system is used). + * Inspired from HBaseClusterTestCase, part of hbase-0.20.3-test.jar. + * + * @author Apache Directory Project + * @version $Rev$, $Date$ + */ +public class EmbeddedHBase +{ + + private static final Logger LOG = LoggerFactory.getLogger( EmbeddedHBase.class ); + + protected File workingDirectory; + protected HBaseConfiguration conf; + protected MiniHBaseCluster cluster; + protected MiniZooKeeperCluster zooKeeperCluster; + + + /** + * Creates a new instance of EmbeddedHBase. + * Note that the configuration is modified to use the local file system + * rather than an DFS. + * + * @param workingDirectory the working directory + * @param conf the HBase configuration + * @throws Exception + */ + public EmbeddedHBase( File workingDirectory, HBaseConfiguration conf ) throws Exception + { + this.workingDirectory = workingDirectory; + this.conf = conf; + + // use local filesystem + conf.set( "fs.default.name", "file:///" ); + // zookeeper working directory + conf.set( "test.build.data", new File( workingDirectory, "zookeeper" ).getAbsolutePath() ); + // hbase working directory + conf.set( HConstants.HBASE_DIR, new File( workingDirectory, "hbase" ).toURI().toURL().toString() ); + } + + + /** + * Starts the embedded HBase cluster. + * + * @throws Exception the exception + */ + public void start() throws Exception + { + try + { + // Note that this is done before we create the MiniHBaseCluster because we + // need to edit the config to add the ZooKeeper servers. + zooKeeperCluster = new MiniZooKeeperCluster(); + int clientPort = this.zooKeeperCluster.startup( workingDirectory ); + conf.set( "hbase.zookeeper.property.clientPort", Integer.toString( clientPort ) ); + + // start the mini cluster + cluster = new MiniHBaseCluster( conf, 1 ); + + } + catch ( Exception e ) + { + LOG.error( "Exception in start.", e ); + if ( cluster != null ) + { + cluster.shutdown(); + } + if ( zooKeeperCluster != null ) + { + zooKeeperCluster.shutdown(); + } + throw e; + } + } + + + /** + * Shutdown the embedded HBase cluster. + * + * @throws Exception the exception + */ + public void shutdown() throws Exception + { + try + { + // open the META table now to ensure cluster is running before shutdown. + new HTable( conf, HConstants.META_TABLE_NAME ); + + HConnectionManager.deleteConnectionInfo( conf, true ); + if ( this.cluster != null ) + { + this.cluster.shutdown(); + this.zooKeeperCluster.shutdown(); + } + } + catch ( Exception e ) + { + LOG.warn( "Exception in shutdown.", e ); + } + } + +} Copied: directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/HBaseDistributedPartition.java (from r902620, directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/HBasePartition.java) URL: http://svn.apache.org/viewvc/directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/HBaseDistributedPartition.java?p2=directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/HBaseDistributedPartition.java&p1=directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/HBasePartition.java&r1=902620&r2=905194&rev=905194&view=diff ============================================================================== --- directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/HBasePartition.java (original) +++ directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/HBaseDistributedPartition.java Mon Feb 1 07:08:23 2010 @@ -20,14 +20,27 @@ package org.apache.directory.server.core.partition.hbase; +import java.util.HashSet; import java.util.Map; +import java.util.Set; +import org.apache.directory.server.constants.ApacheSchemaConstants; import org.apache.directory.server.core.entry.ServerEntry; import org.apache.directory.server.core.partition.Partition; +import org.apache.directory.server.core.partition.hbase.index.AbstractHBaseIndex; +import org.apache.directory.server.core.partition.hbase.index.HBaseAliasIndex; +import org.apache.directory.server.core.partition.hbase.index.HBaseNdnIndex; +import org.apache.directory.server.core.partition.hbase.index.HBaseOneAliasIndex; +import org.apache.directory.server.core.partition.hbase.index.HBaseOneLevelIndex; +import org.apache.directory.server.core.partition.hbase.index.HBasePresenceIndex; +import org.apache.directory.server.core.partition.hbase.index.HBaseSubAliasIndex; +import org.apache.directory.server.core.partition.hbase.index.HBaseSubLevelIndex; +import org.apache.directory.server.core.partition.hbase.index.HBaseUserRowIndex; import org.apache.directory.server.core.partition.hbase.xdbmext.ExtendedCursorBuilder; import org.apache.directory.server.core.partition.hbase.xdbmext.ExtendedEvaluatorBuilder; import org.apache.directory.server.core.partition.hbase.xdbmext.ExtendedOptimizer; import org.apache.directory.server.xdbm.AbstractXdbmPartition; +import org.apache.directory.server.xdbm.Index; import org.apache.directory.server.xdbm.search.impl.DefaultSearchEngine; import org.apache.directory.shared.ldap.schema.normalizers.OidNormalizer; @@ -35,17 +48,18 @@ /** * A {@link Partition} that stores entries in * Hadoop HBase database. + * A running HBase cluster is required and must be configured in hbase-site.xml. * * @author Apache Directory Project * @version $Rev$, $Date$ */ -public class HBasePartition extends AbstractXdbmPartition +public class HBaseDistributedPartition extends AbstractXdbmPartition { /** * Creates a new instance of HBasePartition. */ - public HBasePartition() + public HBaseDistributedPartition() { super( new HBaseStore() ); } @@ -75,6 +89,123 @@ store.setName( id ); ( ( HBaseStore ) store ).setTablePrefix( "apacheds" + "_" + id + "_" ); + Set> userIndices = new HashSet>(); + + for ( Index obj : getIndexedAttributes() ) + { + AbstractHBaseIndex index; + + if ( AbstractHBaseIndex.class.isAssignableFrom( obj.getClass() ) ) + { + index = ( AbstractHBaseIndex ) obj; + } + else + { + index = new HBaseUserRowIndex(); + index.setAttributeId( obj.getAttributeId() ); + index.setCacheSize( obj.getCacheSize() ); + } + + // set the store + index.setStore( ( HBaseStore ) store ); + + // normalize the attributeId to oid + String oid = schemaManager.getAttributeTypeRegistry().getOidByName( index.getAttributeId() ); + index.setAttributeId( oid ); + + if ( oid.equals( ApacheSchemaConstants.APACHE_ALIAS_AT_OID ) ) + { + if ( index instanceof HBaseAliasIndex ) + { + store.setAliasIndex( ( HBaseAliasIndex ) index ); + } + else + { + throw new IllegalStateException( "apacheAlias index must be of type HBaseAliasIndex" ); + } + } + else if ( oid.equals( ApacheSchemaConstants.APACHE_ONE_ALIAS_AT_OID ) ) + { + if ( index instanceof HBaseOneAliasIndex ) + { + store.setOneAliasIndex( ( HBaseOneAliasIndex ) index ); + } + else + { + throw new IllegalStateException( "apacheOneAlias index must be of type HBaseOneAliasIndex" ); + } + } + else if ( oid.equals( ApacheSchemaConstants.APACHE_SUB_ALIAS_AT_OID ) ) + { + if ( index instanceof HBaseSubAliasIndex ) + { + store.setSubAliasIndex( ( HBaseSubAliasIndex ) index ); + } + else + { + throw new IllegalStateException( "apacheSubAlias index must be of type HBaseSubAliasIndex" ); + } + } + else if ( oid.equals( ApacheSchemaConstants.APACHE_EXISTENCE_AT_OID ) ) + { + if ( index instanceof HBasePresenceIndex ) + { + store.setPresenceIndex( ( HBasePresenceIndex ) index ); + } + else + { + throw new IllegalStateException( "apacheExistence index must be of type HBasePresenceIndex" ); + } + } + else if ( oid.equals( ApacheSchemaConstants.APACHE_ONE_LEVEL_AT_OID ) ) + { + if ( index instanceof HBaseOneLevelIndex ) + { + store.setOneLevelIndex( ( HBaseOneLevelIndex ) index ); + } + else + { + throw new IllegalStateException( "apacheOneLevel index must be of type HBaseOneLevelIndex" ); + } + } + else if ( oid.equals( ApacheSchemaConstants.APACHE_SUB_LEVEL_AT_OID ) ) + { + if ( index instanceof HBaseSubLevelIndex ) + { + store.setSubLevelIndex( ( HBaseSubLevelIndex ) index ); + } + else + { + throw new IllegalStateException( "apacheOneLevel index must be of type HBaseSubLevelIndex" ); + } + } + else if ( oid.equals( ApacheSchemaConstants.APACHE_N_DN_AT_OID ) ) + { + if ( index instanceof HBaseNdnIndex ) + { + store.setNdnIndex( ( HBaseNdnIndex ) index ); + } + else + { + throw new IllegalStateException( "apacheNdn index must be of type HBaseNdnIndex" ); + } + } + + else if ( oid.equals( ApacheSchemaConstants.APACHE_UP_DN_AT_OID ) ) + { + throw new IllegalStateException( "apacheUpdn index not supported" ); + } + + // TODO: handle objectClass, entryCSN, entryUUID as system indices? + + else + { + userIndices.add( index ); + } + + } + store.setUserIndices( userIndices ); + store.init( schemaManager ); } Added: directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/HBaseEmbeddedPartition.java URL: http://svn.apache.org/viewvc/directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/HBaseEmbeddedPartition.java?rev=905194&view=auto ============================================================================== --- directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/HBaseEmbeddedPartition.java (added) +++ directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/HBaseEmbeddedPartition.java Mon Feb 1 07:08:23 2010 @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +package org.apache.directory.server.core.partition.hbase; + + +import org.apache.directory.server.core.partition.Partition; + + +/** + * A {@link Partition} that stores entries in an embedded + * Hadoop HBase database. + * + * @author Apache Directory Project + * @version $Rev$, $Date$ + */ +public class HBaseEmbeddedPartition extends HBaseDistributedPartition +{ + + /** + * Creates a new instance of HBaseEmbeddedPartition. + */ + public HBaseEmbeddedPartition() + { + super(); + } + + + // ------------------------------------------------------------------------ + // L I F E C Y C L E M E T H O D S + // ------------------------------------------------------------------------ + + /** + * {@inheritDoc} + */ + public void initialize() throws Exception + { + // setting the working directory triggers the store to + // start up an embedded HBase server. + store.setWorkingDirectory( getPartitionDir() ); + + super.initialize(); + } + +} Modified: directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/HBaseStore.java URL: http://svn.apache.org/viewvc/directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/HBaseStore.java?rev=905194&r1=905193&r2=905194&view=diff ============================================================================== --- directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/HBaseStore.java (original) +++ directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/HBaseStore.java Mon Feb 1 07:08:23 2010 @@ -36,19 +36,15 @@ import org.apache.directory.server.core.partition.hbase.index.HBaseOneAliasIndex; import org.apache.directory.server.core.partition.hbase.index.HBaseOneLevelIndex; import org.apache.directory.server.core.partition.hbase.index.HBasePresenceIndex; +import org.apache.directory.server.core.partition.hbase.index.HBaseSubAliasIndex; import org.apache.directory.server.core.partition.hbase.index.HBaseSubLevelIndex; -import org.apache.directory.server.core.partition.hbase.index.HBaseUserColumnIndex; -import org.apache.directory.server.core.partition.hbase.index.HBaseUserRowIndex; -import org.apache.directory.server.core.partition.hbase.table.HBaseColumnIndexTable; -import org.apache.directory.server.core.partition.hbase.table.HBaseIndexTable; +import org.apache.directory.server.core.partition.hbase.index.HBaseUserIndex; +import org.apache.directory.server.core.partition.hbase.table.HBaseIndexTableBase; import org.apache.directory.server.core.partition.hbase.table.HBaseMasterTable; -import org.apache.directory.server.core.partition.hbase.table.HBasePresenceIndexTable; -import org.apache.directory.server.core.partition.hbase.table.HBaseRowIndexTable; import org.apache.directory.server.xdbm.Index; import org.apache.directory.server.xdbm.IndexCursor; import org.apache.directory.server.xdbm.IndexNotFoundException; import org.apache.directory.server.xdbm.Store; -import org.apache.directory.shared.ldap.constants.SchemaConstants; import org.apache.directory.shared.ldap.entry.EntryAttribute; import org.apache.directory.shared.ldap.entry.Modification; import org.apache.directory.shared.ldap.entry.ModificationOperation; @@ -57,10 +53,13 @@ import org.apache.directory.shared.ldap.name.RDN; import org.apache.directory.shared.ldap.schema.AttributeType; import org.apache.directory.shared.ldap.schema.SchemaManager; +import org.apache.hadoop.hbase.HBaseConfiguration; /** - * The {@link Store} implementation of the {@link HBasePartition}. + * The {@link Store} implementation of the HBase partition. + * If the working directory is set an embedded HBase cluster is used. + * Otherwise a running HBase cluster is required and must be configured in hbase-site.xml * * @author Apache Directory Project * @version $Rev$, $Date$ @@ -75,17 +74,17 @@ private String name; private int cacheSize; private SchemaManager schemaManager; - // private String id; + private EmbeddedHBase embeddedHBase; + private File workingDirectory; + private HBaseConfiguration configuration; private HBaseMasterTable masterTable; - private Map indexTables = new HashMap(); - private Map presenceIndexTables = new HashMap(); - private Map> userIndices = new HashMap>(); - - private HBaseNdnIndex ndnIndex; - private HBaseOneLevelIndex oneLevelIndex; - private HBaseSubLevelIndex subLevelIndex; + private Map> userIndices = new HashMap>(); + private Index presenceIndex; + private Index ndnIndex; + private Index oneLevelIndex; + private Index subLevelIndex; private Index aliasIndex; private Index oneAliasIndex; private Index subAliasIndex; @@ -93,64 +92,78 @@ public void init( SchemaManager schemaManager ) throws Exception { + this.configuration = new HBaseConfiguration(); + if ( workingDirectory != null ) + { + // note that the configuration is modified to use the local file system + this.embeddedHBase = new EmbeddedHBase( workingDirectory, configuration ); + this.embeddedHBase.start(); + } + this.schemaManager = schemaManager; this.suffixDn = new LdapDN( suffix ) .normalize( schemaManager.getAttributeTypeRegistry().getNormalizerMapping() ); - // TODO: make indexed attributes configurable - String[] userIndexAttributes = - { "objectClass", "dc", "o", "ou", "cn", "uid", "entryCSN" }; - for ( String userIndexAttribute : userIndexAttributes ) - { - String oid = schemaManager.getAttributeTypeRegistry().getOidByName( userIndexAttribute ); - // TODO: make configurable - if ( oid.equals( SchemaConstants.CN_AT_OID ) || oid.equals( SchemaConstants.UID_AT_OID ) ) - { - userIndices.put( oid, new HBaseUserColumnIndex( oid, ( HBaseColumnIndexTable ) getIndexTable( oid ), - this ) ); - } - else - { - userIndices.put( oid, new HBaseUserRowIndex( oid, ( HBaseRowIndexTable ) getIndexTable( oid ), this ) ); - } + // setup system indices + if ( presenceIndex == null ) + { + presenceIndex = new HBasePresenceIndex( this ); + } + if ( ndnIndex == null ) + { + ndnIndex = new HBaseNdnIndex( this ); + } + if ( oneLevelIndex == null ) + { + oneLevelIndex = new HBaseOneLevelIndex( this ); + } + if ( subLevelIndex == null ) + { + subLevelIndex = new HBaseSubLevelIndex( this ); } - - ndnIndex = new HBaseNdnIndex( this ); - oneLevelIndex = new HBaseOneLevelIndex( this ); - subLevelIndex = new HBaseSubLevelIndex( this ); - // TODO: implement alias - aliasIndex = new HBaseAliasIndex(); - oneAliasIndex = new HBaseOneAliasIndex(); - subAliasIndex = new HBaseOneAliasIndex(); + if ( aliasIndex == null ) + { + aliasIndex = new HBaseAliasIndex(); + } + if ( oneAliasIndex == null ) + { + oneAliasIndex = new HBaseOneAliasIndex(); + } + if ( subAliasIndex == null ) + { + subAliasIndex = new HBaseSubAliasIndex(); + } } public void destroy() throws Exception { - if ( masterTable != null ) + if ( embeddedHBase != null ) { - masterTable.destroy(); - masterTable = null; + embeddedHBase.shutdown(); + embeddedHBase = null; } - if ( indexTables != null ) + if ( masterTable != null ) { - for ( HBaseIndexTable indexTable : indexTables.values() ) - { - indexTable.destroy(); - } - indexTables.clear(); - indexTables = null; + masterTable.close(); + masterTable = null; } - if ( presenceIndexTables != null ) + + for ( Index userIndex : userIndices.values() ) { - for ( HBasePresenceIndexTable presenceIndexTable : presenceIndexTables.values() ) - { - presenceIndexTable.destroy(); - } - presenceIndexTables.clear(); - presenceIndexTables = null; + userIndex.close(); } + userIndices.clear(); + + presenceIndex.close(); + ndnIndex.close(); + oneLevelIndex.close(); + subLevelIndex.close(); + aliasIndex.close(); + oneAliasIndex.close(); + subAliasIndex.close(); + schemaManager = null; } @@ -167,8 +180,12 @@ String oid = getAttributeTypeOid( attr ); if ( userIndices.containsKey( oid ) ) { - getIndexTable( oid ).add( attribute, id ); - getPresenceIndexTable( oid ).add( id ); + HBaseUserIndex index = userIndices.get( oid ); + for ( Value value : attribute ) + { + index.add( value.getBytes(), id ); + } + getPresenceIndex().add( oid, id ); } } } @@ -178,56 +195,14 @@ { if ( masterTable == null ) { - masterTable = new HBaseMasterTable( getSchemaManager(), getSuffix(), getTablePrefix() ); + masterTable = new HBaseMasterTable( this ); } return masterTable; } - public HBaseIndexTable getIndexTable( String attributeTypeOid ) throws Exception - { - if ( indexTables.containsKey( attributeTypeOid ) ) - { - return indexTables.get( attributeTypeOid ); - } - else - { - HBaseIndexTable indexTable; - // TODO: make configurable - if ( attributeTypeOid.equals( SchemaConstants.CN_AT_OID ) - || attributeTypeOid.equals( SchemaConstants.UID_AT_OID ) ) - { - indexTable = new HBaseColumnIndexTable( attributeTypeOid, schemaManager, tablePrefix ); - } - else - { - indexTable = new HBaseRowIndexTable( attributeTypeOid, schemaManager, tablePrefix ); - } - indexTables.put( attributeTypeOid, indexTable ); - return indexTable; - } - } - - - public HBasePresenceIndexTable getPresenceIndexTable( String attributeTypeOid ) throws Exception - { - if ( presenceIndexTables.containsKey( attributeTypeOid ) ) - { - return presenceIndexTables.get( attributeTypeOid ); - } - else - { - HBasePresenceIndexTable presenceIndexTable = new HBasePresenceIndexTable( attributeTypeOid, schemaManager, - tablePrefix ); - presenceIndexTables.put( attributeTypeOid, presenceIndexTable ); - return presenceIndexTable; - } - } - - public Long getEntryId( String dn ) throws Exception { - //System.out.println(System.currentTimeMillis() + " getEntryId() dn=" + dn); // TODO: is dn already normalized? LdapDN normDn = new LdapDN( dn ).normalize( schemaManager.getAttributeTypeRegistry().getNormalizerMapping() ); Long id = getMasterTable().fetchId( normDn ); @@ -237,16 +212,15 @@ public ServerEntry lookup( Long id ) throws Exception { - //System.out.println(System.currentTimeMillis() + " lookup() id=" + id); ServerEntry entry = getMasterTable().fetchEntry( id ); - //System.out.println(" --> " + entry.getDn()); return entry; } - public void addIndex( Index arg0 ) throws Exception + @SuppressWarnings("unchecked") + public void addIndex( Index index ) { - throw new UnsupportedOperationException(); + this.userIndices.put( index.getAttributeId(), ( HBaseUserIndex ) index ); } @@ -268,8 +242,12 @@ String oid = getAttributeTypeOid( attr ); if ( userIndices.containsKey( oid ) ) { - getIndexTable( oid ).delete( attribute, id ); - getPresenceIndexTable( oid ).delete( id ); + HBaseUserIndex index = userIndices.get( oid ); + for ( Value value : attribute ) + { + index.drop( value.getBytes(), id ); + } + getPresenceIndex().drop( oid, id ); } } @@ -370,7 +348,7 @@ public Index getPresenceIndex() { - return new HBasePresenceIndex( this ); + return presenceIndex; } @@ -456,7 +434,7 @@ public File getWorkingDirectory() { - throw new UnsupportedOperationException(); + return workingDirectory; } @@ -494,7 +472,6 @@ { Long id = getEntryId( dn.getNormName() ); ServerEntry entry = lookup( id ); - //System.out.println("before modify: " + entry); // TODO: quick and dirty implementation to update the indices // remove old indices @@ -504,8 +481,12 @@ String oid = getAttributeTypeOid( attr ); if ( userIndices.containsKey( oid ) ) { - getIndexTable( oid ).delete( attribute, id ); - getPresenceIndexTable( oid ).delete( id ); + HBaseUserIndex index = userIndices.get( oid ); + for ( Value value : attribute ) + { + index.drop( value.getBytes(), id ); + } + getPresenceIndex().drop( oid, id ); } } @@ -569,12 +550,14 @@ String oid = getAttributeTypeOid( attr ); if ( userIndices.containsKey( oid ) ) { - getIndexTable( oid ).add( attribute, id ); - getPresenceIndexTable( oid ).add( id ); + HBaseUserIndex index = userIndices.get( oid ); + for ( Value value : attribute ) + { + index.add( value.getBytes(), id ); + } + getPresenceIndex().add( oid, id ); } } - - //System.out.println("after modify: " + entry); } @@ -602,9 +585,9 @@ } - public void setAliasIndex( Index arg0 ) throws Exception + public void setAliasIndex( Index aliasIndex ) throws Exception { - throw new UnsupportedOperationException(); + this.aliasIndex = aliasIndex; } @@ -632,9 +615,9 @@ } - public void setNdnIndex( Index arg0 ) throws Exception + public void setNdnIndex( Index ndnIndex ) throws Exception { - throw new UnsupportedOperationException(); + this.ndnIndex = ndnIndex; } @@ -644,21 +627,21 @@ } - public void setOneAliasIndex( Index arg0 ) throws Exception + public void setOneAliasIndex( Index oneAliasIndex ) throws Exception { - throw new UnsupportedOperationException(); + this.oneAliasIndex = oneAliasIndex; } - public void setOneLevelIndex( Index arg0 ) throws Exception + public void setOneLevelIndex( Index oneLevelIndex ) throws Exception { - throw new UnsupportedOperationException(); + this.oneLevelIndex = oneLevelIndex; } - public void setPresenceIndex( Index arg0 ) throws Exception + public void setPresenceIndex( Index presenceIndex ) throws Exception { - throw new UnsupportedOperationException(); + this.presenceIndex = presenceIndex; } @@ -668,15 +651,15 @@ } - public void setSubAliasIndex( Index arg0 ) throws Exception + public void setSubAliasIndex( Index subAliasIndex ) throws Exception { - throw new UnsupportedOperationException(); + this.subAliasIndex = subAliasIndex; } - public void setSubLevelIndex( Index arg0 ) throws Exception + public void setSubLevelIndex( Index subLevelIndex ) throws Exception { - throw new UnsupportedOperationException(); + this.subLevelIndex = subLevelIndex; } @@ -698,15 +681,19 @@ } - public void setUserIndices( Set> arg0 ) + public void setUserIndices( Set> userIndices ) { - throw new UnsupportedOperationException(); + this.userIndices.clear(); + for ( Index index : userIndices ) + { + addIndex( index ); + } } - public void setWorkingDirectory( File arg0 ) + public void setWorkingDirectory( File workingDirectory ) { - throw new UnsupportedOperationException(); + this.workingDirectory = workingDirectory; } @@ -744,4 +731,10 @@ { return schemaManager; } + + + public HBaseConfiguration getConfiguration() + { + return configuration; + } } Modified: directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/cursor/HBasePresenceIndexCursor.java URL: http://svn.apache.org/viewvc/directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/cursor/HBasePresenceIndexCursor.java?rev=905194&r1=905193&r2=905194&view=diff ============================================================================== --- directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/cursor/HBasePresenceIndexCursor.java (original) +++ directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/cursor/HBasePresenceIndexCursor.java Mon Feb 1 07:08:23 2010 @@ -21,7 +21,6 @@ import org.apache.directory.server.core.entry.ServerEntry; -import org.apache.directory.server.core.partition.hbase.HBaseStore; import org.apache.directory.server.core.partition.hbase.table.HBasePresenceIndexTable; import org.apache.directory.server.xdbm.ForwardIndexEntry; import org.apache.hadoop.hbase.client.Result; @@ -38,14 +37,13 @@ */ public class HBasePresenceIndexCursor extends BaseHBaseIndexCursor { + private HBasePresenceIndexTable presenceIndexTable; - private String attributeTypeOid; - - public HBasePresenceIndexCursor( String attributeTypeOid, HBaseStore store ) + public HBasePresenceIndexCursor( HBasePresenceIndexTable presenceIndexTable ) { - super( store ); - this.attributeTypeOid = attributeTypeOid; + super( null ); + this.presenceIndexTable = presenceIndexTable; } @@ -54,14 +52,13 @@ { if ( scanner == null ) { - HBasePresenceIndexTable indexTable = store.getPresenceIndexTable( attributeTypeOid ); byte[] start; byte[] stop; - start = indexTable.getPresenceKey( HBasePresenceIndexTable.VALUE_SCAN_FIRST_ENTRYID ); - stop = indexTable.getPresenceKey( HBasePresenceIndexTable.VALUE_SCAN_LAST_ENTRYID ); + start = presenceIndexTable.getPresenceKey( HBasePresenceIndexTable.VALUE_SCAN_FIRST_ENTRYID ); + stop = presenceIndexTable.getPresenceKey( HBasePresenceIndexTable.VALUE_SCAN_LAST_ENTRYID ); Scan s = new Scan( start, stop ); s.addFamily( HBasePresenceIndexTable.INFO_FAMILY ); - scanner = indexTable.getScanner( s ); + scanner = presenceIndexTable.getScanner( s ); iterator = scanner.iterator(); } @@ -73,11 +70,6 @@ currentEntry = new ForwardIndexEntry(); currentEntry.setId( Bytes.toLong( id ) ); - //currentEntry.setValue( null ); - - // ServerEntry entry = store.getId2entryTable().fetchEntry( currentId ); - // currentEntry.setObject( entry ); - // System.out.println("presenceIndex: next() " + entry.getDn().getUpName()); return true; } return false; Modified: directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/index/AbstractHBaseIndex.java URL: http://svn.apache.org/viewvc/directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/index/AbstractHBaseIndex.java?rev=905194&r1=905193&r2=905194&view=diff ============================================================================== --- directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/index/AbstractHBaseIndex.java (original) +++ directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/index/AbstractHBaseIndex.java Mon Feb 1 07:08:23 2010 @@ -22,6 +22,7 @@ import java.io.File; +import org.apache.directory.server.core.partition.hbase.HBaseStore; import org.apache.directory.server.xdbm.Index; import org.apache.directory.server.xdbm.IndexCursor; import org.apache.directory.shared.ldap.cursor.Cursor; @@ -38,7 +39,21 @@ public abstract class AbstractHBaseIndex implements Index { - private int cacheSize; + protected int cacheSize = DEFAULT_INDEX_CACHE_SIZE; + protected String attributeId; + protected HBaseStore store; + + + public HBaseStore getStore() + { + return store; + } + + + public void setStore( HBaseStore store ) + { + this.store = store; + } public void add( K attrVal, Long id ) throws Exception @@ -139,7 +154,7 @@ public String getAttributeId() { - throw new UnsupportedOperationException(); + return attributeId; } @@ -241,7 +256,7 @@ public void setAttributeId( String attributeId ) { - throw new UnsupportedOperationException(); + this.attributeId = attributeId; } Modified: directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/index/HBaseNdnIndex.java URL: http://svn.apache.org/viewvc/directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/index/HBaseNdnIndex.java?rev=905194&r1=905193&r2=905194&view=diff ============================================================================== --- directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/index/HBaseNdnIndex.java (original) +++ directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/index/HBaseNdnIndex.java Mon Feb 1 07:08:23 2010 @@ -36,18 +36,19 @@ public class HBaseNdnIndex extends AbstractHBaseIndex { - private HBaseStore store; - - + public HBaseNdnIndex() + { + } + + public HBaseNdnIndex( HBaseStore store ) { - this.store = store; + setStore( store ); } public void close() throws Exception { - // TODO } Modified: directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/index/HBaseOneLevelIndex.java URL: http://svn.apache.org/viewvc/directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/index/HBaseOneLevelIndex.java?rev=905194&r1=905193&r2=905194&view=diff ============================================================================== --- directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/index/HBaseOneLevelIndex.java (original) +++ directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/index/HBaseOneLevelIndex.java Mon Feb 1 07:08:23 2010 @@ -39,12 +39,13 @@ IndexFilteringExtension { - private HBaseStore store; - + public HBaseOneLevelIndex() + { + } public HBaseOneLevelIndex( HBaseStore store ) { - this.store = store; + setStore( store ); } Modified: directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/index/HBasePresenceIndex.java URL: http://svn.apache.org/viewvc/directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/index/HBasePresenceIndex.java?rev=905194&r1=905193&r2=905194&view=diff ============================================================================== --- directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/index/HBasePresenceIndex.java (original) +++ directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/index/HBasePresenceIndex.java Mon Feb 1 07:08:23 2010 @@ -20,9 +20,13 @@ package org.apache.directory.server.core.partition.hbase.index; +import java.util.HashMap; +import java.util.Map; + import org.apache.directory.server.core.entry.ServerEntry; import org.apache.directory.server.core.partition.hbase.HBaseStore; import org.apache.directory.server.core.partition.hbase.cursor.HBasePresenceIndexCursor; +import org.apache.directory.server.core.partition.hbase.table.HBasePresenceIndexTable; import org.apache.directory.server.xdbm.IndexCursor; @@ -35,25 +39,35 @@ */ public class HBasePresenceIndex extends AbstractHBaseIndex { - private HBaseStore store; + + private Map presenceIndexTables = new HashMap(); + + + public HBasePresenceIndex() + { + } public HBasePresenceIndex( HBaseStore store ) { - this.store = store; + setStore( store ); } public void close() throws Exception { + for ( HBasePresenceIndexTable table : presenceIndexTables.values() ) + { + table.close(); + } + presenceIndexTables.clear(); } @Override public int count( String attributeType ) throws Exception { - int count = store.getPresenceIndexTable( attributeType ).count(); - //System.out.println(attributeType + " -> count=" + count); + int count = getPresenceIndexTable( attributeType ).count(); return count; } @@ -61,14 +75,44 @@ @Override public boolean forward( String attributeTypeOid, Long entryId ) throws Exception { - return store.getPresenceIndexTable( attributeTypeOid ).exists( entryId ); + return getPresenceIndexTable( attributeTypeOid ).exists( entryId ); } @Override public IndexCursor forwardCursor( String key ) throws Exception { - return new HBasePresenceIndexCursor( key, store ); + return new HBasePresenceIndexCursor( getPresenceIndexTable( key ) ); + } + + + @Override + public void add( String attributeTypeOid, Long id ) throws Exception + { + getPresenceIndexTable( attributeTypeOid ).add( id ); + } + + + @Override + public void drop( String attributeTypeOid, Long id ) throws Exception + { + getPresenceIndexTable( attributeTypeOid ).drop( id ); + } + + + private HBasePresenceIndexTable getPresenceIndexTable( String attributeTypeOid ) throws Exception + { + if ( presenceIndexTables.containsKey( attributeTypeOid ) ) + { + return presenceIndexTables.get( attributeTypeOid ); + } + else + { + HBasePresenceIndexTable presenceIndexTable = new HBasePresenceIndexTable( attributeTypeOid, store + .getSchemaManager(), store.getTablePrefix(), store.getConfiguration(), getCacheSize() ); + presenceIndexTables.put( attributeTypeOid, presenceIndexTable ); + return presenceIndexTable; + } } } Added: directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/index/HBaseSubAliasIndex.java URL: http://svn.apache.org/viewvc/directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/index/HBaseSubAliasIndex.java?rev=905194&view=auto ============================================================================== --- directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/index/HBaseSubAliasIndex.java (added) +++ directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/index/HBaseSubAliasIndex.java Mon Feb 1 07:08:23 2010 @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +package org.apache.directory.server.core.partition.hbase.index; + + +import org.apache.directory.server.core.entry.ServerEntry; +import org.apache.directory.server.xdbm.EmptyIndexCursor; +import org.apache.directory.server.xdbm.IndexCursor; + + +/** + * HBase specific implementation of the alias index. + * + * TODO: This is just a stub, needs to be implemented. + * + * @author Apache Directory Project + * @version $Rev$, $Date$ + */ +public class HBaseSubAliasIndex extends AbstractHBaseIndex +{ + + public void close() throws Exception + { + } + + + @Override + public boolean forward( Long attrVal, Long id ) throws Exception + { + // TODO + return false; + } + + + @Override + public IndexCursor forwardCursor( Long key ) throws Exception + { + return new EmptyIndexCursor(); + } +} Modified: directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/index/HBaseSubLevelIndex.java URL: http://svn.apache.org/viewvc/directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/index/HBaseSubLevelIndex.java?rev=905194&r1=905193&r2=905194&view=diff ============================================================================== --- directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/index/HBaseSubLevelIndex.java (original) +++ directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/index/HBaseSubLevelIndex.java Mon Feb 1 07:08:23 2010 @@ -39,12 +39,14 @@ IndexFilteringExtension { - private HBaseStore store; + public HBaseSubLevelIndex() + { + } public HBaseSubLevelIndex( HBaseStore store ) { - this.store = store; + setStore( store ); } Modified: directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/index/HBaseUserColumnIndex.java URL: http://svn.apache.org/viewvc/directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/index/HBaseUserColumnIndex.java?rev=905194&r1=905193&r2=905194&view=diff ============================================================================== --- directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/index/HBaseUserColumnIndex.java (original) +++ directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/index/HBaseUserColumnIndex.java Mon Feb 1 07:08:23 2010 @@ -21,7 +21,6 @@ import org.apache.directory.server.core.entry.ServerEntry; -import org.apache.directory.server.core.partition.hbase.HBaseStore; import org.apache.directory.server.core.partition.hbase.cursor.HBaseUserColumnIndexCursor; import org.apache.directory.server.core.partition.hbase.table.HBaseColumnIndexTable; import org.apache.directory.server.xdbm.IndexCursor; @@ -29,8 +28,8 @@ /** - * HBase specific implementation of an user index that reads - * candidates from treeInfo column family. + * HBase specific implementation of an user index that stores + * candidates within the treeInfo column family. * * @author Apache Directory Project * @version $Rev$, $Date$ @@ -38,29 +37,54 @@ public class HBaseUserColumnIndex extends HBaseUserIndex { - public HBaseUserColumnIndex( String attributeTypeOid, HBaseColumnIndexTable indexTable, HBaseStore store ) + private HBaseColumnIndexTable indexTable; + + + public HBaseUserColumnIndex() { - super( attributeTypeOid, indexTable, store ); + } + + + @Override + public void close() throws Exception + { + super.close(); + if ( indexTable != null ) + { + indexTable.close(); + indexTable = null; + } } @Override public IndexCursor forwardCursor( Object value ) throws Exception { - return new HBaseUserColumnIndexCursor( attributeTypeOid, value, indexTable, store ); + return new HBaseUserColumnIndexCursor( getAttributeId(), value, getIndexTable(), store ); } @Override public IndexCursor forwardCursor() throws Exception { - return new HBaseUserColumnIndexCursor( attributeTypeOid, indexTable, store ); + return new HBaseUserColumnIndexCursor( getAttributeId(), getIndexTable(), store ); } public IndexCursor forwardSubstringCursor( SubstringNode node ) throws Exception { - return new HBaseUserColumnIndexCursor( attributeTypeOid, node, indexTable, store ); + return new HBaseUserColumnIndexCursor( getAttributeId(), node, getIndexTable(), store ); } + + @Override + protected HBaseColumnIndexTable getIndexTable() throws Exception + { + if ( indexTable == null ) + { + indexTable = new HBaseColumnIndexTable( getAttributeId(), store.getSchemaManager(), store.getTablePrefix(), + store.getConfiguration(), getCacheSize() ); + } + return indexTable; + } } Modified: directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/index/HBaseUserIndex.java URL: http://svn.apache.org/viewvc/directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/index/HBaseUserIndex.java?rev=905194&r1=905193&r2=905194&view=diff ============================================================================== --- directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/index/HBaseUserIndex.java (original) +++ directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/index/HBaseUserIndex.java Mon Feb 1 07:08:23 2010 @@ -23,12 +23,12 @@ import java.util.regex.Pattern; import org.apache.directory.server.core.entry.ServerEntry; -import org.apache.directory.server.core.partition.hbase.HBaseStore; import org.apache.directory.server.core.partition.hbase.cursor.HBaseUserIndexReverseCursor; import org.apache.directory.server.core.partition.hbase.table.HBaseIndexTableBase; import org.apache.directory.server.core.partition.hbase.xdbmext.IndexSubstringExtension; import org.apache.directory.server.xdbm.IndexCursor; import org.apache.directory.shared.ldap.filter.SubstringNode; +import org.apache.hadoop.hbase.util.Bytes; /** @@ -40,16 +40,9 @@ public abstract class HBaseUserIndex extends AbstractHBaseIndex implements IndexSubstringExtension { - protected T indexTable; - protected HBaseStore store; - protected String attributeTypeOid; - - protected HBaseUserIndex( String attributeTypeOid, T indexTable, HBaseStore store ) + protected HBaseUserIndex() { - this.attributeTypeOid = attributeTypeOid; - this.indexTable = indexTable; - this.store = store; } @@ -61,7 +54,7 @@ @Override public int count() throws Exception { - int count = store.getPresenceIndexTable( attributeTypeOid ).count(); + int count = store.getPresenceIndex().count( getAttributeId() ); return count; } @@ -69,7 +62,7 @@ @Override public int count( Object value ) throws Exception { - int count = indexTable.count( value ); + int count = getIndexTable().count( value ); return count; } @@ -95,14 +88,14 @@ @Override public boolean forward( Object value, Long id ) throws Exception { - return indexTable.exists( value, id ); + return getIndexTable().exists( value, id ); } @Override public IndexCursor reverseCursor( Long id ) throws Exception { - return new HBaseUserIndexReverseCursor( attributeTypeOid, id, store ); + return new HBaseUserIndexReverseCursor( getAttributeId(), id, store ); } @@ -110,7 +103,7 @@ public boolean reverseLessOrEq( Long id, Object attrVal ) throws Exception { String testValue = ( String ) attrVal; - HBaseUserIndexReverseCursor cursor = new HBaseUserIndexReverseCursor( attributeTypeOid, id, store ); + HBaseUserIndexReverseCursor cursor = new HBaseUserIndexReverseCursor( getAttributeId(), id, store ); boolean match = false; while ( cursor.next() ) @@ -133,7 +126,7 @@ public boolean reverseGreaterOrEq( Long id, Object attrVal ) throws Exception { String testValue = ( String ) attrVal; - HBaseUserIndexReverseCursor cursor = new HBaseUserIndexReverseCursor( attributeTypeOid, id, store ); + HBaseUserIndexReverseCursor cursor = new HBaseUserIndexReverseCursor( getAttributeId(), id, store ); boolean match = false; while ( cursor.next() ) @@ -195,4 +188,35 @@ throw new UnsupportedOperationException(); } + + @Override + public void add( Object value, Long id ) throws Exception + { + if ( value instanceof byte[] ) + { + getIndexTable().add( ( byte[] ) value, id ); + } + else + { + getIndexTable().add( Bytes.toBytes( ( String ) value ), id ); + } + } + + + @Override + public void drop( Object value, Long id ) throws Exception + { + if ( value instanceof byte[] ) + { + getIndexTable().drop( ( byte[] ) value, id ); + } + else + { + getIndexTable().drop( Bytes.toBytes( ( String ) value ), id ); + } + } + + + protected abstract T getIndexTable() throws Exception; + } Modified: directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/index/HBaseUserRowIndex.java URL: http://svn.apache.org/viewvc/directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/index/HBaseUserRowIndex.java?rev=905194&r1=905193&r2=905194&view=diff ============================================================================== --- directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/index/HBaseUserRowIndex.java (original) +++ directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/index/HBaseUserRowIndex.java Mon Feb 1 07:08:23 2010 @@ -21,7 +21,6 @@ import org.apache.directory.server.core.entry.ServerEntry; -import org.apache.directory.server.core.partition.hbase.HBaseStore; import org.apache.directory.server.core.partition.hbase.cursor.HBaseUserRowIndexCursor; import org.apache.directory.server.core.partition.hbase.table.HBaseRowIndexTable; import org.apache.directory.server.xdbm.IndexCursor; @@ -29,8 +28,8 @@ /** - * HBase specific implementation of an user index that reads - * candidates from the row key. + * HBase specific implementation of an user index that stores + * candidates within the row key. * * @author Apache Directory Project * @version $Rev$, $Date$ @@ -38,29 +37,54 @@ public class HBaseUserRowIndex extends HBaseUserIndex { - public HBaseUserRowIndex( String attributeTypeOid, HBaseRowIndexTable indexTable, HBaseStore store ) + private HBaseRowIndexTable indexTable; + + + public HBaseUserRowIndex() { - super( attributeTypeOid, indexTable, store ); + } + + + @Override + public void close() throws Exception + { + super.close(); + if ( indexTable != null ) + { + indexTable.close(); + indexTable = null; + } } @Override public IndexCursor forwardCursor( Object value ) throws Exception { - return new HBaseUserRowIndexCursor( attributeTypeOid, value, indexTable, store ); + return new HBaseUserRowIndexCursor( getAttributeId(), value, getIndexTable(), store ); } @Override public IndexCursor forwardCursor() throws Exception { - return new HBaseUserRowIndexCursor( attributeTypeOid, indexTable, store ); + return new HBaseUserRowIndexCursor( getAttributeId(), getIndexTable(), store ); } public IndexCursor forwardSubstringCursor( SubstringNode node ) throws Exception { - return new HBaseUserRowIndexCursor( attributeTypeOid, node, indexTable, store ); + return new HBaseUserRowIndexCursor( getAttributeId(), node, getIndexTable(), store ); } + + @Override + protected HBaseRowIndexTable getIndexTable() throws Exception + { + if ( indexTable == null ) + { + indexTable = new HBaseRowIndexTable( getAttributeId(), store.getSchemaManager(), store.getTablePrefix(), + store.getConfiguration(), getCacheSize() ); + } + return indexTable; + } } Modified: directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/table/HBaseColumnIndexTable.java URL: http://svn.apache.org/viewvc/directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/table/HBaseColumnIndexTable.java?rev=905194&r1=905193&r2=905194&view=diff ============================================================================== --- directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/table/HBaseColumnIndexTable.java (original) +++ directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/table/HBaseColumnIndexTable.java Mon Feb 1 07:08:23 2010 @@ -28,6 +28,7 @@ import org.apache.directory.shared.ldap.schema.SchemaManager; import org.apache.directory.shared.ldap.util.Base64; import org.apache.directory.shared.ldap.util.ByteBuffer; +import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.client.Delete; import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.Put; @@ -53,11 +54,11 @@ private Cache infoCache; - public HBaseColumnIndexTable( String attributeTypeOid, SchemaManager schemaManager, String tablePrefix ) - throws Exception + public HBaseColumnIndexTable( String attributeTypeOid, SchemaManager schemaManager, String tablePrefix, + HBaseConfiguration configuration, int cacheSize ) throws Exception { - super( attributeTypeOid, schemaManager, tablePrefix ); - this.infoCache = new Cache(); + super( attributeTypeOid, schemaManager, tablePrefix, configuration, cacheSize ); + this.infoCache = new Cache( cacheSize ); } @@ -204,7 +205,7 @@ } - protected void add( byte[] value, Long id ) throws Exception + public void add( byte[] value, Long id ) throws Exception { // exact match (attribute=value): #value -> count, id // check first if the index already exists because we won't increment the index count @@ -226,10 +227,12 @@ // TODO: optimize - don't need to clear the ẃhole cache infoCache.clear(); + countCache.clear(); + existsCache.clear(); } - protected void delete( byte[] value, Long id ) throws Exception + public void drop( byte[] value, Long id ) throws Exception { // exact match (attribute=value): #value -> count, id // check first if the index exists because we won't decrement the index count otherwise @@ -250,6 +253,8 @@ // TODO: optimize - don't need to clear the ẃhole cache infoCache.clear(); + countCache.clear(); + existsCache.clear(); } class Info Modified: directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/table/HBaseIndexTable.java URL: http://svn.apache.org/viewvc/directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/table/HBaseIndexTable.java?rev=905194&r1=905193&r2=905194&view=diff ============================================================================== --- directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/table/HBaseIndexTable.java (original) +++ directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/table/HBaseIndexTable.java Mon Feb 1 07:08:23 2010 @@ -20,7 +20,6 @@ package org.apache.directory.server.core.partition.hbase.table; -import org.apache.directory.shared.ldap.entry.EntryAttribute; import org.apache.hadoop.hbase.client.ResultScanner; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.util.Bytes; @@ -48,13 +47,7 @@ { ( byte ) 0xFF }; - public abstract void destroy() throws Exception; - - - public abstract void add( EntryAttribute attribute, Long id ) throws Exception; - - - public abstract void delete( EntryAttribute attribute, Long id ) throws Exception; + public abstract void close() throws Exception; public abstract int count( Object value ) throws Exception; Modified: directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/table/HBaseIndexTableBase.java URL: http://svn.apache.org/viewvc/directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/table/HBaseIndexTableBase.java?rev=905194&r1=905193&r2=905194&view=diff ============================================================================== --- directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/table/HBaseIndexTableBase.java (original) +++ directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/table/HBaseIndexTableBase.java Mon Feb 1 07:08:23 2010 @@ -21,7 +21,6 @@ import org.apache.directory.server.core.partition.hbase.Cache; -import org.apache.directory.shared.ldap.entry.EntryAttribute; import org.apache.directory.shared.ldap.entry.Value; import org.apache.directory.shared.ldap.entry.client.ClientBinaryValue; import org.apache.directory.shared.ldap.schema.AttributeType; @@ -47,23 +46,25 @@ protected String indexTableName; private HTablePool indexTablePool; protected SchemaManager schemaManager; + protected HBaseConfiguration configuration; protected Cache countCache; protected Cache existsCache; - public HBaseIndexTableBase( String attributeTypeOid, SchemaManager schemaManager, String tablePrefix ) - throws Exception + public HBaseIndexTableBase( String attributeTypeOid, SchemaManager schemaManager, String tablePrefix, + HBaseConfiguration configuration, int cacheSize ) throws Exception { this.attributeTypeOid = attributeTypeOid; this.schemaManager = schemaManager; + this.configuration = configuration; String name = schemaManager.getGlobalOidRegistry().getPrimaryName( attributeTypeOid ); this.indexTableName = tablePrefix + "index_" + name; - this.countCache = new Cache(); - this.existsCache = new Cache(); + this.countCache = new Cache( cacheSize ); + this.existsCache = new Cache( cacheSize ); } - public void destroy() throws Exception + public void close() throws Exception { } @@ -74,36 +75,10 @@ } - public void add( EntryAttribute attribute, Long id ) throws Exception - { - for ( Value value : attribute ) - { - add( value.getBytes(), id ); - } - - // TODO: optimize - don't need to clear the ẃhole cache - countCache.clear(); - existsCache.clear(); - } - - - protected abstract void add( byte[] value, Long id ) throws Exception; - - - public void delete( EntryAttribute attribute, Long id ) throws Exception - { - for ( Value value : attribute ) - { - delete( value.getBytes(), id ); - } - - // TODO: optimize - don't need to clear the ẃhole cache - countCache.clear(); - existsCache.clear(); - } + public abstract void add( byte[] value, Long id ) throws Exception; - protected abstract void delete( byte[] value, Long id ) throws Exception; + public abstract void drop( byte[] value, Long id ) throws Exception; protected byte[] getNormalized( Object value ) throws Exception @@ -142,7 +117,6 @@ { if ( indexTablePool == null ) { - HBaseConfiguration configuration = HBaseTableHelper.getHBaseConfiguration(); // ensure table is created HBaseTableHelper.createTable( configuration, indexTableName, INFO_FAMILY ); indexTablePool = new HTablePool( configuration, 16 ); Modified: directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/table/HBaseMasterTable.java URL: http://svn.apache.org/viewvc/directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/table/HBaseMasterTable.java?rev=905194&r1=905193&r2=905194&view=diff ============================================================================== --- directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/table/HBaseMasterTable.java (original) +++ directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/table/HBaseMasterTable.java Mon Feb 1 07:08:23 2010 @@ -32,6 +32,7 @@ import org.apache.directory.server.core.entry.ServerEntry; import org.apache.directory.server.core.entry.ServerStringValue; import org.apache.directory.server.core.partition.hbase.Cache; +import org.apache.directory.server.core.partition.hbase.HBaseStore; import org.apache.directory.shared.ldap.entry.EntryAttribute; import org.apache.directory.shared.ldap.entry.Value; import org.apache.directory.shared.ldap.name.LdapDN; @@ -86,10 +87,9 @@ private static final String MASTER_TABLE = "master"; private static final String TREE_TABLE = "tree"; - // private HBaseStore store; + private HBaseConfiguration configuration; private SchemaManager schemaManager; private LdapDN suffixDn; - // private String tablePrefix; private HTablePool masterTablePool; private String masterTableName; @@ -98,40 +98,48 @@ private MasterTreeInfo suffixMti; /** id -> master tree info */ - private Cache mtiCache = new Cache(); + private Cache mtiCache; /** master tree info -> id */ - private Cache idCache = new Cache(); + private Cache idCache; /** id -> one level count */ - private Cache oneLevelCountCache = new Cache(); + private Cache oneLevelCountCache; /** id -> sub level count */ - private Cache subLevelCountCache = new Cache(); + private Cache subLevelCountCache; /** id -> DN */ - private Cache dnCache = new Cache(); + private Cache dnCache; /** id -> entry */ - private Cache entryCache = new Cache(); + private Cache entryCache; - public HBaseMasterTable( SchemaManager schemaManager, LdapDN suffixDn, String tablePrefix ) + public HBaseMasterTable( HBaseStore store ) { - // this.store = store; - // this.schemaManager = store.getSchemaManager(); - // this.suffixDn = store.getSuffix(); - // this.tablePrefix = store.getTablePrefix(); - this.schemaManager = schemaManager; - this.suffixDn = suffixDn; - // this.tablePrefix = tablePrefix; - this.masterTableName = tablePrefix + MASTER_TABLE; - this.treeTableName = tablePrefix + TREE_TABLE; + this.schemaManager = store.getSchemaManager(); + this.suffixDn = store.getSuffix(); + this.configuration = store.getConfiguration(); + this.masterTableName = store.getTablePrefix() + MASTER_TABLE; + this.treeTableName = store.getTablePrefix() + TREE_TABLE; this.suffixMti = new MasterTreeInfo( ROOT_ID, suffixDn.getNormName(), null ); + + this.entryCache = new Cache( store.getCacheSize() ); + + int dnCacheSize = Math.max( store.getCacheSize(), store.getNdnIndex().getCacheSize() ); + this.dnCache = new Cache( dnCacheSize ); + this.mtiCache = new Cache( dnCacheSize ); + this.idCache = new Cache( dnCacheSize ); + + this.oneLevelCountCache = new Cache( Math.max( store.getCacheSize(), store.getOneLevelIndex() + .getCacheSize() ) ); + this.subLevelCountCache = new Cache( Math.max( store.getCacheSize(), store.getSubLevelIndex() + .getCacheSize() ) ); } - public void destroy() throws Exception + public void close() throws Exception { } @@ -595,8 +603,6 @@ { if ( masterTablePool == null ) { - HBaseConfiguration configuration = HBaseTableHelper.getHBaseConfiguration(); - // ensure table is created HBaseTableHelper.createTable( configuration, masterTableName, TREE_INFO_FAMILY, UP_ATTRIBUTES_FAMILY ); masterTablePool = new HTablePool( configuration, 16 ); } @@ -608,7 +614,6 @@ { if ( treeTablePool == null ) { - HBaseConfiguration configuration = HBaseTableHelper.getHBaseConfiguration(); HBaseTableHelper.createTable( configuration, treeTableName, TREE_INFO_FAMILY, NORM_ATTRIBUTES_FAMILY ); treeTablePool = new HTablePool( configuration, 16 ); } Modified: directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/table/HBasePresenceIndexTable.java URL: http://svn.apache.org/viewvc/directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/table/HBasePresenceIndexTable.java?rev=905194&r1=905193&r2=905194&view=diff ============================================================================== --- directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/table/HBasePresenceIndexTable.java (original) +++ directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/table/HBasePresenceIndexTable.java Mon Feb 1 07:08:23 2010 @@ -57,41 +57,29 @@ protected String indexTableName; private HTablePool indexTablePool; private SchemaManager schemaManager; + private HBaseConfiguration configuration; private String attributeTypeOid; private Cache countCache; - public HBasePresenceIndexTable( String attributeTypeOid, SchemaManager schemaManager, String tablePrefix ) - throws Exception + public HBasePresenceIndexTable( String attributeTypeOid, SchemaManager schemaManager, String tablePrefix, + HBaseConfiguration configuration, int cacheSize ) throws Exception { this.attributeTypeOid = attributeTypeOid; this.schemaManager = schemaManager; + this.configuration = configuration; String name = schemaManager.getGlobalOidRegistry().getPrimaryName( attributeTypeOid ); this.indexTableName = tablePrefix + "index_" + name; - this.countCache = new Cache(); + this.countCache = new Cache( cacheSize ); } - public void destroy() throws Exception + public void close() throws Exception { } - public void add( Long id ) throws Exception - { - add( Bytes.toBytes( id ) ); - countCache.clear(); - } - - - public void delete( Long id ) throws Exception - { - delete( Bytes.toBytes( id ) ); - countCache.clear(); - } - - /** * Gets the index equals key. * The key has the following syntax: @@ -162,31 +150,33 @@ } - private void add( byte[] entryId ) throws Exception + public void add( Long entryId ) throws Exception { // presence (attribute=*): * -> id // check first if the index already exists because we won't increment the index count - byte[] presenceRow = getPresenceKey( entryId ); + byte[] presenceRow = getPresenceKey( Bytes.toBytes( entryId ) ); Get presenceGet = new Get( presenceRow ); if ( !HBaseTableHelper.exists( getIndexTablePool(), indexTableName, presenceGet ) ) { // get+put+put is not atomic! Put presencePut = new Put( presenceRow ); presencePut.setWriteToWAL( false ); - presencePut.add( INFO_FAMILY, ID_QUALIFIER, entryId ); + presencePut.add( INFO_FAMILY, ID_QUALIFIER, Bytes.toBytes( entryId ) ); HBaseTableHelper.put( getIndexTablePool(), indexTableName, presencePut ); // increment existence count: attribute: -> count HBaseTableHelper.increment( getIndexTablePool(), indexTableName, COUNT_ROW, INFO_FAMILY, COUNT_QUALIFIER ); } + + countCache.clear(); } - private void delete( byte[] entryId ) throws Exception + public void drop( Long entryId ) throws Exception { // presence (attribute=*): * -> id // check first if the index exists because we won't decrement the index count otherwise - byte[] presenceRow = getPresenceKey( entryId ); + byte[] presenceRow = getPresenceKey( Bytes.toBytes( entryId ) ); Get presenceGet = new Get( presenceRow ); if ( HBaseTableHelper.exists( getIndexTablePool(), indexTableName, presenceGet ) ) { @@ -196,6 +186,8 @@ // decrement existence count: attribute: -> count HBaseTableHelper.decrement( getIndexTablePool(), indexTableName, COUNT_ROW, INFO_FAMILY, COUNT_QUALIFIER ); } + + countCache.clear(); } @@ -203,7 +195,6 @@ { if ( indexTablePool == null ) { - HBaseConfiguration configuration = HBaseTableHelper.getHBaseConfiguration(); // ensure table is created HBaseTableHelper.createTable( configuration, indexTableName, INFO_FAMILY ); indexTablePool = new HTablePool( configuration, 16 ); Modified: directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/table/HBaseRowIndexTable.java URL: http://svn.apache.org/viewvc/directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/table/HBaseRowIndexTable.java?rev=905194&r1=905193&r2=905194&view=diff ============================================================================== --- directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/table/HBaseRowIndexTable.java (original) +++ directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/table/HBaseRowIndexTable.java Mon Feb 1 07:08:23 2010 @@ -24,6 +24,7 @@ import org.apache.directory.shared.ldap.schema.SchemaManager; import org.apache.directory.shared.ldap.util.Base64; import org.apache.directory.shared.ldap.util.ByteBuffer; +import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.client.Delete; import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.Put; @@ -46,10 +47,10 @@ private static final Logger LOG = LoggerFactory.getLogger( HBaseRowIndexTable.class ); - public HBaseRowIndexTable( String attributeTypeOid, SchemaManager schemaManager, String tablePrefix ) - throws Exception + public HBaseRowIndexTable( String attributeTypeOid, SchemaManager schemaManager, String tablePrefix, + HBaseConfiguration configuration, int cacheSize ) throws Exception { - super( attributeTypeOid, schemaManager, tablePrefix ); + super( attributeTypeOid, schemaManager, tablePrefix, configuration, cacheSize ); } @@ -177,7 +178,7 @@ } - protected void add( byte[] value, Long id ) throws Exception + public void add( byte[] value, Long id ) throws Exception { // exact match (attribute=value): =value -> id, value // check first if the index already exists because we won't increment the index count @@ -196,10 +197,14 @@ HBaseTableHelper.increment( getIndexTablePool(), indexTableName, exactCountRow, INFO_FAMILY, COUNT_QUALIFIER ); } + + // TODO: optimize - don't need to clear the ẃhole cache + countCache.clear(); + existsCache.clear(); } - protected void delete( byte[] value, Long id ) throws Exception + public void drop( byte[] value, Long id ) throws Exception { // exact match (attribute=value): =value -> id // check first if the index exists because we won't decrement the index count otherwise @@ -216,6 +221,10 @@ COUNT_QUALIFIER ); // TODO: delete column if count is 0? } + + // TODO: optimize - don't need to clear the ẃhole cache + countCache.clear(); + existsCache.clear(); } } Modified: directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/table/HBaseTableHelper.java URL: http://svn.apache.org/viewvc/directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/table/HBaseTableHelper.java?rev=905194&r1=905193&r2=905194&view=diff ============================================================================== --- directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/table/HBaseTableHelper.java (original) +++ directory/sandbox/seelmann/hbase-partition/src/main/java/org/apache/directory/server/core/partition/hbase/table/HBaseTableHelper.java Mon Feb 1 07:08:23 2010 @@ -378,17 +378,6 @@ } - /** - * Gets the configuration. - * - * @return the configuration - */ - public static HBaseConfiguration getHBaseConfiguration() - { - return new HBaseConfiguration(); - } - - private static void logStack( int start ) { StackTraceElement[] stackTrace = Thread.currentThread().getStackTrace();