manifoldcf-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kwri...@apache.org
Subject svn commit: r1208116 - in /incubator/lcf/branches/CONNECTORS-286/warthog/src: main/java/org/apache/warthog/common/ main/java/org/apache/warthog/tablestore/ test/java/org/apache/warthog/tests/
Date Tue, 29 Nov 2011 22:36:28 GMT
Author: kwright
Date: Tue Nov 29 22:36:26 2011
New Revision: 1208116

URL: http://svn.apache.org/viewvc?rev=1208116&view=rev
Log:
Finish the 'native hash/equals' experiment, which shows roughly a factor of two improvement.

Added:
    incubator/lcf/branches/CONNECTORS-286/warthog/src/test/java/org/apache/warthog/tests/InMemAtomicNativeKeyValueStore.java
  (with props)
    incubator/lcf/branches/CONNECTORS-286/warthog/src/test/java/org/apache/warthog/tests/InMemNativeTransactionImpl.java
  (with props)
    incubator/lcf/branches/CONNECTORS-286/warthog/src/test/java/org/apache/warthog/tests/InMemNativeTransactionalStoreImpl.java
  (with props)
Modified:
    incubator/lcf/branches/CONNECTORS-286/warthog/src/main/java/org/apache/warthog/common/LongValue.java
    incubator/lcf/branches/CONNECTORS-286/warthog/src/main/java/org/apache/warthog/tablestore/GeneralIDFactoryKey.java
    incubator/lcf/branches/CONNECTORS-286/warthog/src/main/java/org/apache/warthog/tablestore/IndexAccessor.java
    incubator/lcf/branches/CONNECTORS-286/warthog/src/main/java/org/apache/warthog/tablestore/IndexNodeKey.java
    incubator/lcf/branches/CONNECTORS-286/warthog/src/test/java/org/apache/warthog/tests/MultiThreadTest.java
    incubator/lcf/branches/CONNECTORS-286/warthog/src/test/java/org/apache/warthog/tests/PerformanceTest.java
    incubator/lcf/branches/CONNECTORS-286/warthog/src/test/java/org/apache/warthog/tests/SanityTest.java

Modified: incubator/lcf/branches/CONNECTORS-286/warthog/src/main/java/org/apache/warthog/common/LongValue.java
URL: http://svn.apache.org/viewvc/incubator/lcf/branches/CONNECTORS-286/warthog/src/main/java/org/apache/warthog/common/LongValue.java?rev=1208116&r1=1208115&r2=1208116&view=diff
==============================================================================
--- incubator/lcf/branches/CONNECTORS-286/warthog/src/main/java/org/apache/warthog/common/LongValue.java
(original)
+++ incubator/lcf/branches/CONNECTORS-286/warthog/src/main/java/org/apache/warthog/common/LongValue.java
Tue Nov 29 22:36:26 2011
@@ -58,7 +58,7 @@ public class LongValue implements WHValu
 
   public int hashCode()
   {
-    return (((int)value) << 5) ^ (((int)value) >> 3);
+    return (int)((value << 5) ^ (value >> 3));
   }
   
   public boolean equals(Object o)

Modified: incubator/lcf/branches/CONNECTORS-286/warthog/src/main/java/org/apache/warthog/tablestore/GeneralIDFactoryKey.java
URL: http://svn.apache.org/viewvc/incubator/lcf/branches/CONNECTORS-286/warthog/src/main/java/org/apache/warthog/tablestore/GeneralIDFactoryKey.java?rev=1208116&r1=1208115&r2=1208116&view=diff
==============================================================================
--- incubator/lcf/branches/CONNECTORS-286/warthog/src/main/java/org/apache/warthog/tablestore/GeneralIDFactoryKey.java
(original)
+++ incubator/lcf/branches/CONNECTORS-286/warthog/src/main/java/org/apache/warthog/tablestore/GeneralIDFactoryKey.java
Tue Nov 29 22:36:26 2011
@@ -50,4 +50,14 @@ public class GeneralIDFactoryKey impleme
     return true;
   }
 
+  public int hashCode()
+  {
+    return 15335;
+  }
+  
+  public boolean equals(Object o)
+  {
+    return o.getClass() == getClass();
+  }
+  
 }

Modified: incubator/lcf/branches/CONNECTORS-286/warthog/src/main/java/org/apache/warthog/tablestore/IndexAccessor.java
URL: http://svn.apache.org/viewvc/incubator/lcf/branches/CONNECTORS-286/warthog/src/main/java/org/apache/warthog/tablestore/IndexAccessor.java?rev=1208116&r1=1208115&r2=1208116&view=diff
==============================================================================
--- incubator/lcf/branches/CONNECTORS-286/warthog/src/main/java/org/apache/warthog/tablestore/IndexAccessor.java
(original)
+++ incubator/lcf/branches/CONNECTORS-286/warthog/src/main/java/org/apache/warthog/tablestore/IndexAccessor.java
Tue Nov 29 22:36:26 2011
@@ -156,7 +156,12 @@ public class IndexAccessor implements WH
         if (columnIndex + 1 == indexColumns.length)
           queue.pushChaseEquals(rowID);
         else
-          queue.pushNeedsAnalysis(index.readIndexNodeColumnNode(rowID,indexColumns[columnIndex]),columnIndex+1);
+        {
+          LongValue childRowID = index.readIndexNodeColumnNode(rowID,indexColumns[columnIndex]);
+          if (childRowID == null)
+            throw new WHConcurrencyException();
+          queue.pushNeedsAnalysis(childRowID,columnIndex+1);
+        }
       }
       if ((criteria & IndexCriteria.SIGNAL_LESSER) != 0)
       {

Modified: incubator/lcf/branches/CONNECTORS-286/warthog/src/main/java/org/apache/warthog/tablestore/IndexNodeKey.java
URL: http://svn.apache.org/viewvc/incubator/lcf/branches/CONNECTORS-286/warthog/src/main/java/org/apache/warthog/tablestore/IndexNodeKey.java?rev=1208116&r1=1208115&r2=1208116&view=diff
==============================================================================
--- incubator/lcf/branches/CONNECTORS-286/warthog/src/main/java/org/apache/warthog/tablestore/IndexNodeKey.java
(original)
+++ incubator/lcf/branches/CONNECTORS-286/warthog/src/main/java/org/apache/warthog/tablestore/IndexNodeKey.java
Tue Nov 29 22:36:26 2011
@@ -79,4 +79,21 @@ public class IndexNodeKey implements WHK
       key.columnName.equals(columnName);
   }
 
+  public int hashCode()
+  {
+    return ((int)((indexID << 5) ^ (indexID >> 3))) +
+      ((int)((rowID << 5) ^ (rowID >> 3))) +
+      columnName.hashCode();
+  }
+  
+  public boolean equals(Object o)
+  {
+    if (o.getClass() != getClass())
+      return false;
+    IndexNodeKey other = (IndexNodeKey)o;
+    return other.indexID == indexID &&
+      other.rowID == rowID &&
+      other.columnName.equals(columnName);
+  }
+  
 }

Added: incubator/lcf/branches/CONNECTORS-286/warthog/src/test/java/org/apache/warthog/tests/InMemAtomicNativeKeyValueStore.java
URL: http://svn.apache.org/viewvc/incubator/lcf/branches/CONNECTORS-286/warthog/src/test/java/org/apache/warthog/tests/InMemAtomicNativeKeyValueStore.java?rev=1208116&view=auto
==============================================================================
--- incubator/lcf/branches/CONNECTORS-286/warthog/src/test/java/org/apache/warthog/tests/InMemAtomicNativeKeyValueStore.java
(added)
+++ incubator/lcf/branches/CONNECTORS-286/warthog/src/test/java/org/apache/warthog/tests/InMemAtomicNativeKeyValueStore.java
Tue Nov 29 22:36:26 2011
@@ -0,0 +1,98 @@
+/* $Id$ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.warthog.tests;
+
+import org.apache.warthog.interfaces.*;
+import java.util.*;
+
+/** In-memory key value store, native hash, for testing.
+*/
+public class InMemAtomicNativeKeyValueStore implements WHAtomicKeyValueStore
+{
+  protected Map<WHKey,WHValue> database;
+  
+  /** Constructor */
+  public InMemAtomicNativeKeyValueStore(int initialSize)
+  {
+    database = new HashMap<WHKey,WHValue>(initialSize);
+  }
+  
+  /** Get a value */
+  public WHValue get(WHKey key)
+    throws WHException
+  {
+    synchronized (database)
+    {
+      return database.get(key);
+    }
+  }
+  
+  /** Check a bunch of values atomically for consistency.
+  *@param checkValues is a map of keys/values that must be unchanged.  If any value is
+  * changed, a WHDeadlockException is thrown.
+  */
+  public void check(WHKeyMap checkValues)
+    throws WHException
+  {
+    synchronized (database)
+    {
+      WHKeyIterator iterator = checkValues.iterator();
+      while (iterator.hasNext())
+      {
+        WHKey key = iterator.next();
+        WHValue value = database.get(key);
+        WHValue otherValue = checkValues.get(key);
+	if (value == null && otherValue == null)
+          continue;
+        if (value != null && otherValue != null && value.equals(otherValue))
+          continue;
+        //System.out.println("Type of key in contention = "+key.getClass().getName());
+        throw new WHConcurrencyException();
+      }
+    }
+  }
+  
+  /** Set a bunch of values atomically.
+  *@param checkValues is a map of keys/values that must be unchanged in order for the
+  * commit to proceed.  If these values are detected to have been changed, a WHDeadlockException
+  * will be thrown.  Null values are permitted.
+  *@param setValues is a map of keys to set to specified new values.  A null value implies
removal of
+  * the key.
+  */
+  public void setAll(WHKeyMap checkValues, WHKeyMap setValues)
+    throws WHException
+  {
+    synchronized (database)
+    {
+      check(checkValues);
+      WHKeyIterator iterator = setValues.iterator();
+      while (iterator.hasNext())
+      {
+        WHKey key = iterator.next();
+        WHValue value = setValues.get(key);
+        if (value == null)
+          database.remove(key);
+        else
+          database.put(key,value);
+      }
+    }
+  }
+  
+}

Propchange: incubator/lcf/branches/CONNECTORS-286/warthog/src/test/java/org/apache/warthog/tests/InMemAtomicNativeKeyValueStore.java
------------------------------------------------------------------------------
    svn:eol-style = native

Propchange: incubator/lcf/branches/CONNECTORS-286/warthog/src/test/java/org/apache/warthog/tests/InMemAtomicNativeKeyValueStore.java
------------------------------------------------------------------------------
    svn:keywords = Id

Added: incubator/lcf/branches/CONNECTORS-286/warthog/src/test/java/org/apache/warthog/tests/InMemNativeTransactionImpl.java
URL: http://svn.apache.org/viewvc/incubator/lcf/branches/CONNECTORS-286/warthog/src/test/java/org/apache/warthog/tests/InMemNativeTransactionImpl.java?rev=1208116&view=auto
==============================================================================
--- incubator/lcf/branches/CONNECTORS-286/warthog/src/test/java/org/apache/warthog/tests/InMemNativeTransactionImpl.java
(added)
+++ incubator/lcf/branches/CONNECTORS-286/warthog/src/test/java/org/apache/warthog/tests/InMemNativeTransactionImpl.java
Tue Nov 29 22:36:26 2011
@@ -0,0 +1,163 @@
+/* $Id$ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.warthog.tests;
+
+import org.apache.warthog.interfaces.*;
+import java.util.*;
+
+/** This class implements a transaction that is in progress, completely in memory,
+* using native hash methods.
+*/
+public class InMemNativeTransactionImpl implements WHTransaction
+{
+  protected WHAtomicKeyValueStore underlyingStore;
+  
+  protected KeyMap checkData = new KeyMap();
+  protected KeyMap localData = null;
+
+  /** Constructor */
+  public InMemNativeTransactionImpl(WHAtomicKeyValueStore underlyingStore)
+  {
+    this.underlyingStore = underlyingStore;
+  }
+  
+  /** Set a value.  May be null. */
+  public void put(WHKey key, WHValue value)
+    throws WHException
+  {
+    if (localData == null)
+      localData = new KeyMap();
+    localData.put(key,value);
+  }
+
+  /** Get a value.  Null returned if no value. */
+  public WHValue get(WHKey key)
+    throws WHException
+  {
+    if (localData != null)
+    {
+      if (localData.containsKey(key))
+        return localData.get(key);
+    }
+    if (checkData.containsKey(key))
+      return checkData.get(key);
+    WHValue value = underlyingStore.get(key);
+    checkData.put(key,value);
+    return value;
+  }
+
+  /** Check to see if this transaction has become inconsistent.
+  * If so, a WHDeadlockException is thrown.
+  */
+  public void check()
+    throws WHException
+  {
+    underlyingStore.check(checkData);
+  }
+
+  /** Commit this transaction */
+  public void commit()
+    throws WHException
+  {
+    if (localData != null)
+      underlyingStore.setAll(checkData,localData);
+    else
+      underlyingStore.check(checkData);
+  }
+  
+  /** Abandon this transaction.
+  * This is called as a nicety to free any resources associated with the
+  * transaction.  The implementation should also be robust as far as
+  * freeing resources if this method is NOT called, but might perform
+  * the necessary logic in a finalizer at an arbitrary time.
+  */
+  public void abandon()
+  {
+    // Does nothing so long as the entire temporary transaction is in
+    // memory.
+  }
+
+  /** Local implementation of WHKey/WHValue map */
+  protected static class KeyMap implements WHKeyMap
+  {
+    protected Map<WHKey,WHValue> map = new HashMap<WHKey,WHValue>();
+    
+    public KeyMap()
+    {
+    }
+    
+    /** Get a value from the map.
+    */
+    public WHValue get(WHKey key)
+      throws WHException
+    {
+      return map.get(key);
+    }
+    
+    /** Iterate over the keys in the map.
+    */
+    public WHKeyIterator iterator()
+      throws WHException
+    {
+      return new KeyIterator(map.keySet().iterator());
+    }
+    
+    /** Check if the map contains the specified key.
+    */
+    public boolean containsKey(WHKey key)
+    {
+      return map.containsKey(key);
+    }
+
+    /** Put a value.
+    */
+    public void put(WHKey key, WHValue value)
+    {
+      map.put(key,value);
+    }
+  }
+  
+  /** Key iterator for KeyMap */
+  protected static class KeyIterator implements WHKeyIterator
+  {
+    protected Iterator<WHKey> iterator;
+    
+    public KeyIterator(Iterator<WHKey> iterator)
+    {
+      this.iterator = iterator;
+    }
+    
+    /** Check if there is another value */
+    public boolean hasNext()
+      throws WHException
+    {
+      return iterator.hasNext();
+    }
+    
+    /** Get the next value */
+    public WHKey next()
+      throws WHException
+    {
+      return iterator.next();
+    }
+
+  }
+  
+}

Propchange: incubator/lcf/branches/CONNECTORS-286/warthog/src/test/java/org/apache/warthog/tests/InMemNativeTransactionImpl.java
------------------------------------------------------------------------------
    svn:eol-style = native

Propchange: incubator/lcf/branches/CONNECTORS-286/warthog/src/test/java/org/apache/warthog/tests/InMemNativeTransactionImpl.java
------------------------------------------------------------------------------
    svn:keywords = Id

Added: incubator/lcf/branches/CONNECTORS-286/warthog/src/test/java/org/apache/warthog/tests/InMemNativeTransactionalStoreImpl.java
URL: http://svn.apache.org/viewvc/incubator/lcf/branches/CONNECTORS-286/warthog/src/test/java/org/apache/warthog/tests/InMemNativeTransactionalStoreImpl.java?rev=1208116&view=auto
==============================================================================
--- incubator/lcf/branches/CONNECTORS-286/warthog/src/test/java/org/apache/warthog/tests/InMemNativeTransactionalStoreImpl.java
(added)
+++ incubator/lcf/branches/CONNECTORS-286/warthog/src/test/java/org/apache/warthog/tests/InMemNativeTransactionalStoreImpl.java
Tue Nov 29 22:36:26 2011
@@ -0,0 +1,44 @@
+/* $Id$ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.warthog.tests;
+
+import org.apache.warthog.interfaces.*;
+
+/** Class implementing a native transactional key/value store completely in memory.
+*/
+public class InMemNativeTransactionalStoreImpl implements WHTransactionalStore
+{
+  /** Underlying atomic key/value store */
+  protected WHAtomicKeyValueStore underlyingStore;
+  
+  /** Constructor. */
+  public InMemNativeTransactionalStoreImpl(WHAtomicKeyValueStore underlyingStore)
+  {
+    this.underlyingStore = underlyingStore;
+  }
+  
+  /** Create a transaction. */
+  public WHTransaction createTransaction()
+    throws WHException
+  {
+    return new InMemNativeTransactionImpl(this.underlyingStore);
+  }
+
+}

Propchange: incubator/lcf/branches/CONNECTORS-286/warthog/src/test/java/org/apache/warthog/tests/InMemNativeTransactionalStoreImpl.java
------------------------------------------------------------------------------
    svn:eol-style = native

Propchange: incubator/lcf/branches/CONNECTORS-286/warthog/src/test/java/org/apache/warthog/tests/InMemNativeTransactionalStoreImpl.java
------------------------------------------------------------------------------
    svn:keywords = Id

Modified: incubator/lcf/branches/CONNECTORS-286/warthog/src/test/java/org/apache/warthog/tests/MultiThreadTest.java
URL: http://svn.apache.org/viewvc/incubator/lcf/branches/CONNECTORS-286/warthog/src/test/java/org/apache/warthog/tests/MultiThreadTest.java?rev=1208116&r1=1208115&r2=1208116&view=diff
==============================================================================
--- incubator/lcf/branches/CONNECTORS-286/warthog/src/test/java/org/apache/warthog/tests/MultiThreadTest.java
(original)
+++ incubator/lcf/branches/CONNECTORS-286/warthog/src/test/java/org/apache/warthog/tests/MultiThreadTest.java
Tue Nov 29 22:36:26 2011
@@ -37,9 +37,9 @@ public class MultiThreadTest
     
     int repeatCount = totalInserts/threadCount;
 
-    WHAtomicKeyValueStore store = new InMemAtomicKeyValueStore(16777216);
+    WHAtomicKeyValueStore store = new InMemAtomicNativeKeyValueStore(16777216);
     
-    WHTableStore ts = new TableStore(new InMemTransactionalStoreImpl(store));
+    WHTableStore ts = new TableStore(new InMemNativeTransactionalStoreImpl(store));
 
     ts.beginTransaction();
     ts.createTable("testtable",new String[]{"colA","colB","colC","colD","colE","colF","colG","colH","colI","colJ"});
@@ -114,9 +114,9 @@ public class MultiThreadTest
     
     int repeatCount = totalCount/threadCount;
 
-    WHAtomicKeyValueStore store = new InMemAtomicKeyValueStore(16777216);
+    WHAtomicKeyValueStore store = new InMemAtomicNativeKeyValueStore(16777216);
     
-    WHTableStore ts = new TableStore(new InMemTransactionalStoreImpl(store));
+    WHTableStore ts = new TableStore(new InMemNativeTransactionalStoreImpl(store));
 
     ts.beginTransaction();
     ts.createTable("testtable",new String[]{"colA","colB","colC","colD","colE","colF","colG","colH","colI","colJ"});

Modified: incubator/lcf/branches/CONNECTORS-286/warthog/src/test/java/org/apache/warthog/tests/PerformanceTest.java
URL: http://svn.apache.org/viewvc/incubator/lcf/branches/CONNECTORS-286/warthog/src/test/java/org/apache/warthog/tests/PerformanceTest.java?rev=1208116&r1=1208115&r2=1208116&view=diff
==============================================================================
--- incubator/lcf/branches/CONNECTORS-286/warthog/src/test/java/org/apache/warthog/tests/PerformanceTest.java
(original)
+++ incubator/lcf/branches/CONNECTORS-286/warthog/src/test/java/org/apache/warthog/tests/PerformanceTest.java
Tue Nov 29 22:36:26 2011
@@ -32,7 +32,7 @@ public class PerformanceTest
   public void rowCreateTest()
     throws Exception
   {
-    WHTableStore ts = new TableStore(new InMemTransactionalStoreImpl(new InMemAtomicKeyValueStore(16777216)));
+    WHTableStore ts = new TableStore(new InMemNativeTransactionalStoreImpl(new InMemAtomicNativeKeyValueStore(16777216)));
 
     ts.beginTransaction();
     ts.createTable("testtable",new String[]{"colA","colB","colC","colD","colE","colF","colG","colH","colI","colJ"});

Modified: incubator/lcf/branches/CONNECTORS-286/warthog/src/test/java/org/apache/warthog/tests/SanityTest.java
URL: http://svn.apache.org/viewvc/incubator/lcf/branches/CONNECTORS-286/warthog/src/test/java/org/apache/warthog/tests/SanityTest.java?rev=1208116&r1=1208115&r2=1208116&view=diff
==============================================================================
--- incubator/lcf/branches/CONNECTORS-286/warthog/src/test/java/org/apache/warthog/tests/SanityTest.java
(original)
+++ incubator/lcf/branches/CONNECTORS-286/warthog/src/test/java/org/apache/warthog/tests/SanityTest.java
Tue Nov 29 22:36:26 2011
@@ -458,6 +458,7 @@ public class SanityTest
   protected WHTableStore createInMemTableStore()
     throws WHException
   {
+    //return new TableStore(new InMemNativeTransactionalStoreImpl(new InMemAtomicNativeKeyValueStore(32768)));
     return new TableStore(new InMemTransactionalStoreImpl(new InMemAtomicKeyValueStore(32768)));
   }
     



Mime
View raw message