hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From st...@apache.org
Subject svn commit: r593665 - in /lucene/hadoop/trunk/src/contrib/hbase: ./ src/java/org/apache/hadoop/hbase/ src/test/org/apache/hadoop/hbase/
Date Fri, 09 Nov 2007 21:57:25 GMT
Author: stack
Date: Fri Nov  9 13:57:24 2007
New Revision: 593665

URL: http://svn.apache.org/viewvc?rev=593665&view=rev
Log:
HADOOP-2161 getRow() is orders of magnitudes slower than get(), even on
rows with one column

Modified:
    lucene/hadoop/trunk/src/contrib/hbase/CHANGES.txt
    lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HMaster.java
    lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HMemcache.java
    lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HRegion.java
    lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HRegionServer.java
    lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HScannerInterface.java
    lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HStore.java
    lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HTable.java
    lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/HBaseTestCase.java
    lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/PerformanceEvaluation.java

Modified: lucene/hadoop/trunk/src/contrib/hbase/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hbase/CHANGES.txt?rev=593665&r1=593664&r2=593665&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/contrib/hbase/CHANGES.txt (original)
+++ lucene/hadoop/trunk/src/contrib/hbase/CHANGES.txt Fri Nov  9 13:57:24 2007
@@ -28,6 +28,8 @@
                problematic.
    HADOOP-2155 Method expecting HBaseConfiguration throws NPE when given Configuration
    HADOOP-2156 BufferUnderflowException for un-named HTableDescriptors
+   HADOOP-2161 getRow() is orders of magnitudes slower than get(), even on rows
+               with one column (Clint Morgan and Stack)
 
   IMPROVEMENTS
     HADOOP-2401 Add convenience put method that takes writable

Modified: lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HMaster.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HMaster.java?rev=593665&r1=593664&r2=593665&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HMaster.java (original)
+++ lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HMaster.java Fri
Nov  9 13:57:24 2007
@@ -3063,7 +3063,7 @@
             (new LocalHBaseCluster(conf)).startup();
           } else {
             Constructor<? extends HMaster> c =
-              masterClass.getConstructor(Configuration.class);
+              masterClass.getConstructor(HBaseConfiguration.class);
             HMaster master = c.newInstance(conf);
             master.start();
           }

Modified: lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HMemcache.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HMemcache.java?rev=593665&r1=593664&r2=593665&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HMemcache.java
(original)
+++ lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HMemcache.java
Fri Nov  9 13:57:24 2007
@@ -235,7 +235,7 @@
           && key.matchesWithoutColumn(itKey)) {
         byte [] val = tailMap.get(itKey);
         results.put(itCol, val);
-      } else if (key.getRow().compareTo(itKey.getRow()) > 0) {
+      } else if (key.getRow().compareTo(itKey.getRow()) < 0) {
         break;
       }
     }

Modified: lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HRegion.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HRegion.java?rev=593665&r1=593664&r2=593665&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HRegion.java (original)
+++ lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HRegion.java Fri
Nov  9 13:57:24 2007
@@ -35,7 +35,6 @@
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.filter.RowFilterInterface;
@@ -1018,13 +1017,11 @@
    */
   TreeMap<Text, byte []> getFull(Text row) throws IOException {
     HStoreKey key = new HStoreKey(row, System.currentTimeMillis());
-
     lock.obtainReadLock();
     try {
       TreeMap<Text, byte []> memResult = memcache.getFull(key);
       for (Text colFamily: stores.keySet()) {
-        HStore targetStore = stores.get(colFamily);
-        targetStore.getFull(key, memResult);
+        this.stores.get(colFamily).getFull(key, memResult);
       }
       return memResult;
     } finally {

Modified: lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HRegionServer.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HRegionServer.java?rev=593665&r1=593664&r2=593665&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HRegionServer.java
(original)
+++ lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HRegionServer.java
Fri Nov  9 13:57:24 2007
@@ -1453,7 +1453,7 @@
               "hbase.master is set to 'local' mode");
           } else {
             Constructor<? extends HRegionServer> c =
-              regionServerClass.getConstructor(Configuration.class);
+              regionServerClass.getConstructor(HBaseConfiguration.class);
             HRegionServer hrs = c.newInstance(conf);
             Thread t = new Thread(hrs);
             t.setName("regionserver" + hrs.server.getListenerAddress());

Modified: lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HScannerInterface.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HScannerInterface.java?rev=593665&r1=593664&r2=593665&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HScannerInterface.java
(original)
+++ lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HScannerInterface.java
Fri Nov  9 13:57:24 2007
@@ -21,6 +21,7 @@
 
 import java.io.Closeable;
 import java.io.IOException;
+import java.util.Iterator;
 import java.util.Map;
 import java.util.SortedMap;
 
@@ -28,7 +29,8 @@
 
 /**
  * HScannerInterface iterates through a set of rows.  It's implemented by
- * several classes.
+ * several classes.  Implements {@link Iterable} but be sure to still call
+ * {@link #close()} when done with your {@link Iterator}
  */
 public interface HScannerInterface extends Closeable,
 Iterable<Map.Entry<HStoreKey, SortedMap<Text, byte []>>> {

Modified: lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HStore.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HStore.java?rev=593665&r1=593664&r2=593665&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HStore.java (original)
+++ lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HStore.java Fri
Nov  9 13:57:24 2007
@@ -1100,7 +1100,7 @@
               }
               results.put(new Text(readcol), readval.get());
               readval = new ImmutableBytesWritable();
-            } else if(key.getRow().compareTo(readkey.getRow()) > 0) {
+            } else if(key.getRow().compareTo(readkey.getRow()) < 0) {
               break;
             }
             

Modified: lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HTable.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HTable.java?rev=593665&r1=593664&r2=593665&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HTable.java (original)
+++ lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HTable.java Fri
Nov  9 13:57:24 2007
@@ -564,13 +564,19 @@
   
   /** 
    * Start an atomic row insertion/update.  No changes are committed until the 
-   * call to commit() returns.
-   * 
-   * A call to abort() will abandon any updates in progress.
+   * call to commit() returns. A call to abort() will abandon any updates in
+   * progress.
    *
    * 
-   * @param row Name of row to start update against.
-   * @return Row lockid.
+   * @param row Name of row to start update against.  Note, choose row names
+   * with care.  Rows are sorted lexicographically (comparison is done
+   * using {@link Text#compareTo(Object)}.  If your keys are numeric,
+   * lexicographic sorting means that 46 sorts AFTER 450 (If you want to use
+   * numerics for keys, zero-pad).
+   * @return Row lock id..
+   * @see #commit(long)
+   * @see #commit(long, long)
+   * @see #abort(long)
    */
   public synchronized long startUpdate(final Text row) {
     checkClosed();

Modified: lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/HBaseTestCase.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/HBaseTestCase.java?rev=593665&r1=593664&r2=593665&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/HBaseTestCase.java
(original)
+++ lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/HBaseTestCase.java
Fri Nov  9 13:57:24 2007
@@ -37,6 +37,8 @@
   protected final static String COLFAMILY_NAME1 = "colfamily1:";
   protected final static String COLFAMILY_NAME2 = "colfamily2:";
   protected final static String COLFAMILY_NAME3 = "colfamily3:";
+  protected static Text [] COLUMNS = new Text [] {new Text(COLFAMILY_NAME1),
+    new Text(COLFAMILY_NAME2), new Text(COLFAMILY_NAME3)};
   protected Path testDir = null;
   protected FileSystem localFs = null;
   protected static final char FIRST_CHAR = 'a';

Modified: lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/PerformanceEvaluation.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/PerformanceEvaluation.java?rev=593665&r1=593664&r2=593665&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/PerformanceEvaluation.java
(original)
+++ lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/PerformanceEvaluation.java
Fri Nov  9 13:57:24 2007
@@ -25,6 +25,7 @@
 import java.text.SimpleDateFormat;
 import java.util.Arrays;
 import java.util.Date;
+import java.util.Formatter;
 import java.util.List;
 import java.util.Random;
 import java.util.TreeMap;
@@ -293,6 +294,7 @@
     protected HBaseAdmin admin;
     protected HTable table;
     protected volatile HBaseConfiguration conf;
+    private Formatter formatter = new Formatter();
     
     Test(final HBaseConfiguration conf, final int startRow,
         final int perClientRunRows, final int totalRows, final Status status) {
@@ -366,10 +368,14 @@
     }
     
     Text getRandomRow() {
-      return new Text(Integer.toString(this.rand.nextInt(Integer.MAX_VALUE) %
+      return new Text(format(this.rand.nextInt(Integer.MAX_VALUE) %
         this.totalRows));
     }
     
+    public Text format(final int i) {
+      return new Text(String.format("%010d", Integer.valueOf(i)));
+    }
+    
     /*
      * Test for individual row.
      * @param i Row index.
@@ -439,7 +445,7 @@
     void testSetup() throws IOException {
       super.testSetup();
       this.testScanner = table.obtainScanner(new Text[] {COLUMN_NAME},
-          new Text(Integer.toString(this.startRow)));
+        format(this.startRow));
     }
     
     @Override
@@ -471,7 +477,7 @@
     
     @Override
     void testRow(final int i) throws IOException {
-      table.get(new Text(Integer.toString(i)), COLUMN_NAME);
+      table.get(format(i), COLUMN_NAME);
     }
 
     @Override
@@ -488,7 +494,7 @@
     
     @Override
     void testRow(final int i) throws IOException {
-      long lockid = table.startUpdate(new Text(Integer.toString(i)));
+      long lockid = table.startUpdate(format(i));
       table.put(lockid, COLUMN_NAME, generateValue());
       table.commit(lockid);
     }



Mime
View raw message