hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From st...@apache.org
Subject svn commit: r592548 - in /lucene/hadoop/trunk/src/contrib/hbase: ./ src/java/org/apache/hadoop/hbase/ src/java/org/apache/hadoop/hbase/shell/ src/test/org/apache/hadoop/hbase/
Date Tue, 06 Nov 2007 20:45:28 GMT
Author: stack
Date: Tue Nov  6 12:45:28 2007
New Revision: 592548

URL: http://svn.apache.org/viewvc?rev=592548&view=rev
Log:
HADOOP-2157 Make Scanners implement Iterable

Modified:
    lucene/hadoop/trunk/src/contrib/hbase/CHANGES.txt
    lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HAbstractScanner.java
    lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HRegion.java
    lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HScannerInterface.java
    lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HTable.java
    lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/shell/SelectCommand.java
    lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestScanner2.java

Modified: lucene/hadoop/trunk/src/contrib/hbase/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hbase/CHANGES.txt?rev=592548&r1=592547&r2=592548&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/contrib/hbase/CHANGES.txt (original)
+++ lucene/hadoop/trunk/src/contrib/hbase/CHANGES.txt Tue Nov  6 12:45:28 2007
@@ -32,8 +32,10 @@
                 (Johan Oskarsson via Stack)
     HADOOP-2088 Make hbase runnable in $HADOOP_HOME/build(/contrib/hbase)
     HADOOP-2126 Use Bob Jenkins' hash for bloom filters
+    HADOOP-2157 Make Scanners implement Iterable
 
-Branch 0.15 (unreleased changes)
+Release 0.15.1
+Branch 0.15
 
   INCOMPATIBLE CHANGES
     HADOOP-1931 Hbase scripts take --ARG=ARG_VALUE when should be like hadoop

Modified: lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HAbstractScanner.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HAbstractScanner.java?rev=592548&r1=592547&r2=592548&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HAbstractScanner.java
(original)
+++ lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HAbstractScanner.java
Tue Nov  6 12:45:28 2007
@@ -20,9 +20,11 @@
 package org.apache.hadoop.hbase;
 
 import java.io.IOException;
+import java.util.Iterator;
 import java.util.SortedMap;
 import java.util.TreeMap;
 import java.util.Vector;
+import java.util.Map.Entry;
 import java.util.regex.Pattern;
 
 import org.apache.commons.logging.Log;
@@ -273,5 +275,10 @@
       }
     }
     return insertedItem;
+  }
+  
+  public Iterator<Entry<HStoreKey, SortedMap<Text, byte[]>>> iterator()
{
+    throw new UnsupportedOperationException("Unimplemented serverside. " +
+      "next(HStoreKey, StortedMap(...) is more efficient");
   }
 }

Modified: lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HRegion.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HRegion.java?rev=592548&r1=592547&r2=592548&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HRegion.java (original)
+++ lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HRegion.java Tue
Nov  6 12:45:28 2007
@@ -22,6 +22,7 @@
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Random;
@@ -29,6 +30,7 @@
 import java.util.TreeMap;
 import java.util.TreeSet;
 import java.util.Vector;
+import java.util.Map.Entry;
 import java.util.concurrent.atomic.AtomicBoolean;
 
 import org.apache.commons.logging.Log;
@@ -1807,6 +1809,11 @@
           closeScanner(i);
         }
       }
+    }
+
+    public Iterator<Entry<HStoreKey, SortedMap<Text, byte[]>>> iterator()
{
+      throw new UnsupportedOperationException("Unimplemented serverside. " +
+        "next(HStoreKey, StortedMap(...) is more efficient");
     }
   }
   

Modified: lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HScannerInterface.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HScannerInterface.java?rev=592548&r1=592547&r2=592548&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HScannerInterface.java
(original)
+++ lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HScannerInterface.java
Tue Nov  6 12:45:28 2007
@@ -19,16 +19,19 @@
  */
 package org.apache.hadoop.hbase;
 
-import org.apache.hadoop.io.Text;
-
+import java.io.Closeable;
 import java.io.IOException;
+import java.util.Map;
 import java.util.SortedMap;
 
+import org.apache.hadoop.io.Text;
+
 /**
  * HScannerInterface iterates through a set of rows.  It's implemented by
  * several classes.
  */
-public interface HScannerInterface {
+public interface HScannerInterface extends Closeable,
+Iterable<Map.Entry<HStoreKey, SortedMap<Text, byte []>>> {
   /**
    * Grab the next row's worth of values. The scanner will return the most
    * recent data value for each row that is not newer than the target time

Modified: lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HTable.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HTable.java?rev=592548&r1=592547&r2=592548&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HTable.java (original)
+++ lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HTable.java Tue
Nov  6 12:45:28 2007
@@ -22,10 +22,12 @@
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.Iterator;
 import java.util.Map;
 import java.util.Random;
 import java.util.SortedMap;
 import java.util.TreeMap;
+import java.util.Map.Entry;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.concurrent.atomic.AtomicReferenceArray;
 
@@ -33,6 +35,8 @@
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.filter.RowFilterInterface;
+import org.apache.hadoop.hbase.filter.StopRowFilter;
+import org.apache.hadoop.hbase.filter.WhileMatchRowFilter;
 import org.apache.hadoop.hbase.io.BatchUpdate;
 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
 import org.apache.hadoop.hbase.util.Writables;
@@ -383,9 +387,8 @@
    * @return scanner
    * @throws IOException
    */
-  public HScannerInterface obtainScanner(Text[] columns,
-      Text startRow) throws IOException {
-    
+  public HScannerInterface obtainScanner(Text[] columns, Text startRow)
+  throws IOException {
     return obtainScanner(columns, startRow, System.currentTimeMillis(), null);
   }
   
@@ -403,9 +406,9 @@
    * @return scanner
    * @throws IOException
    */
-  public HScannerInterface obtainScanner(Text[] columns,
-      Text startRow, long timestamp) throws IOException {
-    
+  public HScannerInterface obtainScanner(Text[] columns, Text startRow,
+      long timestamp)
+  throws IOException {
     return obtainScanner(columns, startRow, timestamp, null);
   }
   
@@ -423,11 +426,60 @@
    * @return scanner
    * @throws IOException
    */
-  public HScannerInterface obtainScanner(Text[] columns,
-      Text startRow, RowFilterInterface filter) throws IOException { 
-    
+  public HScannerInterface obtainScanner(Text[] columns, Text startRow,
+      RowFilterInterface filter)
+  throws IOException { 
     return obtainScanner(columns, startRow, System.currentTimeMillis(), filter);
   }
+
+  /** 
+   * Get a scanner on the current table starting at the specified row and
+   * ending just before <code>stopRow<code>.
+   * Return the specified columns.
+   *
+   * @param columns columns to scan. If column name is a column family, all
+   * columns of the specified column family are returned.  Its also possible
+   * to pass a regex in the column qualifier. A column qualifier is judged to
+   * be a regex if it contains at least one of the following characters:
+   * <code>\+|^&*$[]]}{)(</code>.
+   * @param startRow starting row in table to scan
+   * @param stopRow Row to stop scanning on. Once we hit this row we stop
+   * returning values; i.e. we return the row before this one but not the
+   * <code>stopRow</code> itself.
+   * @return scanner
+   * @throws IOException
+   */
+  public HScannerInterface obtainScanner(final Text[] columns,
+      final Text startRow, final Text stopRow)
+  throws IOException {
+    return obtainScanner(columns, startRow, stopRow,
+      System.currentTimeMillis());
+  }
+
+  /** 
+   * Get a scanner on the current table starting at the specified row and
+   * ending just before <code>stopRow<code>.
+   * Return the specified columns.
+   *
+   * @param columns columns to scan. If column name is a column family, all
+   * columns of the specified column family are returned.  Its also possible
+   * to pass a regex in the column qualifier. A column qualifier is judged to
+   * be a regex if it contains at least one of the following characters:
+   * <code>\+|^&*$[]]}{)(</code>.
+   * @param startRow starting row in table to scan
+   * @param stopRow Row to stop scanning on. Once we hit this row we stop
+   * returning values; i.e. we return the row before this one but not the
+   * <code>stopRow</code> itself.
+   * @param timestamp only return results whose timestamp <= this value
+   * @return scanner
+   * @throws IOException
+   */
+  public HScannerInterface obtainScanner(final Text[] columns,
+      final Text startRow, final Text stopRow, final long timestamp)
+  throws IOException {
+    return obtainScanner(columns, startRow, timestamp,
+      new WhileMatchRowFilter(new StopRowFilter(stopRow)));
+  }
   
   /** 
    * Get a scanner on the current table starting at the specified row.
@@ -447,7 +499,6 @@
   public HScannerInterface obtainScanner(Text[] columns,
       Text startRow, long timestamp, RowFilterInterface filter)
   throws IOException {
-    
     checkClosed();
     return new ClientScanner(columns, startRow, timestamp, filter);
   }
@@ -881,5 +932,45 @@
       this.server = null;
       this.closed = true;
     }
+
+    public Iterator<Entry<HStoreKey, SortedMap<Text, byte[]>>> iterator()
{
+      return new Iterator<Entry<HStoreKey, SortedMap<Text, byte[]>>>()
{
+        HStoreKey key = null;
+        SortedMap<Text, byte []> value = null;
+        
+        public boolean hasNext() {
+          boolean hasNext = false;
+          try {
+            this.key = new HStoreKey();
+            this.value = new TreeMap<Text, byte[]>();
+            hasNext = ClientScanner.this.next(key, value);
+          } catch (IOException e) {
+            throw new RuntimeException(e);
+          }
+          return hasNext;
+        }
+
+        public Entry<HStoreKey, SortedMap<Text, byte[]>> next() {
+          return new Map.Entry<HStoreKey, SortedMap<Text, byte[]>>() {
+            public HStoreKey getKey() {
+              return key;
+            }
+
+            public SortedMap<Text, byte[]> getValue() {
+              return value;
+            }
+
+            public SortedMap<Text, byte[]> setValue(@SuppressWarnings("unused")
+            SortedMap<Text, byte[]> value) {
+              throw new UnsupportedOperationException();
+            }
+          };
+        }
+
+        public void remove() {
+          throw new UnsupportedOperationException();
+        }
+      };
+    }
   }
-}
+}
\ No newline at end of file

Modified: lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/shell/SelectCommand.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/shell/SelectCommand.java?rev=592548&r1=592547&r2=592548&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/shell/SelectCommand.java
(original)
+++ lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/shell/SelectCommand.java
Tue Nov  6 12:45:28 2007
@@ -46,7 +46,6 @@
  * TODO: INTO FILE is not yet implemented.
  */
 public class SelectCommand extends BasicCommand {
-  
   private Text tableName;
   private Text rowKey = new Text("");
   private List<String> columns;
@@ -220,6 +219,8 @@
             break;
           }
         }
+        // Clear results else subsequent results polluted w/ previous finds.
+        results.clear();
       }
       formatter.footer();
       scan.close();

Modified: lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestScanner2.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestScanner2.java?rev=592548&r1=592547&r2=592548&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestScanner2.java
(original)
+++ lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestScanner2.java
Tue Nov  6 12:45:28 2007
@@ -26,15 +26,13 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.SortedMap;
 import java.util.TreeMap;
 import java.util.regex.Pattern;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.MapWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.Writable;
 import org.apache.hadoop.hbase.filter.RegExpRowFilter;
 import org.apache.hadoop.hbase.filter.RowFilterInterface;
 import org.apache.hadoop.hbase.filter.RowFilterSet;
@@ -42,6 +40,9 @@
 import org.apache.hadoop.hbase.filter.WhileMatchRowFilter;
 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
 import org.apache.hadoop.hbase.util.Writables;
+import org.apache.hadoop.io.MapWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.Writable;
 
 /**
  * Additional scanner tests.
@@ -69,13 +70,38 @@
       fail();
     }
   }
+
+  public void testStopRow() throws Exception {
+    Text tableName = new Text(getName());
+    createTable(new HBaseAdmin(this.conf), tableName);
+    HTable table = new HTable(this.conf, tableName);
+    final String lastKey = "aac";
+    addContent(new HTableIncommon(table), FIRST_COLKEY + ":");
+    HScannerInterface scanner =
+      table.obtainScanner(new Text [] {new Text(FIRST_COLKEY + ":")},
+        HConstants.EMPTY_START_ROW, new Text(lastKey));
+    for (Map.Entry<HStoreKey, SortedMap<Text, byte []>> e: scanner) {
+      LOG.info(e.getKey());
+      assertTrue(e.getKey().getRow().toString().compareTo(lastKey) < 0);
+    }
+  }
   
+  public void testIterator() throws Exception {
+    HTable table = new HTable(this.conf, HConstants.ROOT_TABLE_NAME);
+    HScannerInterface scanner =
+      table.obtainScanner(HConstants.COLUMN_FAMILY_ARRAY,
+      HConstants.EMPTY_START_ROW);
+    for (Map.Entry<HStoreKey, SortedMap<Text, byte []>> e: scanner) {
+      assertNotNull(e.getKey());
+      assertNotNull(e.getValue());
+    }
+  }
+
   /**
    * Test getting scanners with regexes for column names.
    * @throws IOException 
    */
   public void testRegexForColumnName() throws IOException {
-    // Setup HClient, ensure that it is running correctly
     HBaseAdmin admin = new HBaseAdmin(conf);
     
     // Setup colkeys to be inserted



Mime
View raw message