hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From st...@apache.org
Subject svn commit: r684952 - in /hadoop/hbase/branches/0.2: CHANGES.txt src/java/org/apache/hadoop/hbase/client/HConnectionManager.java src/java/org/apache/hadoop/hbase/mapred/TableInputFormatBase.java src/test/org/apache/hadoop/hbase/client/TestHTable.java
Date Mon, 11 Aug 2008 21:58:12 GMT
Author: stack
Date: Mon Aug 11 14:58:12 2008
New Revision: 684952

URL: http://svn.apache.org/viewvc?rev=684952&view=rev
Log:
HBASE-729 client region/metadata cache should have a public method for invalidating entries

Modified:
    hadoop/hbase/branches/0.2/CHANGES.txt
    hadoop/hbase/branches/0.2/src/java/org/apache/hadoop/hbase/client/HConnectionManager.java
    hadoop/hbase/branches/0.2/src/java/org/apache/hadoop/hbase/mapred/TableInputFormatBase.java
    hadoop/hbase/branches/0.2/src/test/org/apache/hadoop/hbase/client/TestHTable.java

Modified: hadoop/hbase/branches/0.2/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.2/CHANGES.txt?rev=684952&r1=684951&r2=684952&view=diff
==============================================================================
--- hadoop/hbase/branches/0.2/CHANGES.txt (original)
+++ hadoop/hbase/branches/0.2/CHANGES.txt Mon Aug 11 14:58:12 2008
@@ -1,16 +1,19 @@
 Hbase Change Log
 
-Release 0.3.0 - Unreleased
+Release 0.2.1 - Unreleased
 
   INCOMPATIBLE CHANGES
   BUG FIXES
    HBASE-805   Remove unnecessary getRow overloads in HRS (Jonathan Gray via
                Jim Kellerman) (Fix whitespace changes in HRegionServer)
    HBASE-811   HTD is not fully copyable (Andrew Purtell via Jim Kellerman)
+   HBASE-729   Client region/metadata cache should have a public method for
+               invalidating entries (Andrew Purtell via Stack)
 
   IMPROVEMENTS
    HBASE-801  When a table haven't disable, shell could response in a "user
               friendly" way.
+   HBASE-816  TableMap should survive USE (Andrew Purtell via Stack)
 
   NEW FEATURES
   OPTIMIZATIONS

Modified: hadoop/hbase/branches/0.2/src/java/org/apache/hadoop/hbase/client/HConnectionManager.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.2/src/java/org/apache/hadoop/hbase/client/HConnectionManager.java?rev=684952&r1=684951&r2=684952&view=diff
==============================================================================
--- hadoop/hbase/branches/0.2/src/java/org/apache/hadoop/hbase/client/HConnectionManager.java
(original)
+++ hadoop/hbase/branches/0.2/src/java/org/apache/hadoop/hbase/client/HConnectionManager.java
Mon Aug 11 14:58:12 2008
@@ -371,61 +371,42 @@
       return rowsScanned > 0 && result;
     }
     
+    private class HTableDescriptorFinder 
+    implements MetaScanner.MetaScannerVisitor {
+        byte[] tableName;
+        HTableDescriptor result;
+        public HTableDescriptorFinder(byte[] tableName) {
+          this.tableName = tableName;
+        }
+        public boolean processRow(RowResult rowResult) throws IOException {
+          HRegionInfo info = Writables.getHRegionInfo(
+            rowResult.get(HConstants.COL_REGIONINFO));
+          HTableDescriptor desc = info.getTableDesc();
+          if (Bytes.compareTo(desc.getName(), tableName) == 0) {
+            result = desc;
+            return false;
+          }
+          return true;
+        }
+        HTableDescriptor getResult() {
+          return result;
+        }
+    }
+
     /** {@inheritDoc} */
-    public HTableDescriptor getHTableDescriptor(byte[] tableName)
+    public HTableDescriptor getHTableDescriptor(final byte[] tableName)
     throws IOException {
       if (Bytes.equals(tableName, HConstants.ROOT_TABLE_NAME)) {
         return new UnmodifyableHTableDescriptor(HTableDescriptor.ROOT_TABLEDESC);
       }
-      if (!tableExists(tableName)) {
-        throw new TableNotFoundException(Bytes.toString(tableName));
+      if (Bytes.equals(tableName, HConstants.META_TABLE_NAME)) {
+        return new UnmodifyableHTableDescriptor(HTableDescriptor.META_TABLEDESC);
       }
-      byte[] startKey =
-        HRegionInfo.createRegionName(tableName, null, HConstants.ZEROES);
-      
-      HTableDescriptor result = null;
-      HRegionInfo currentRegion = null;
-      ScannerCallable s = null;
-      while (result == null) {
-        if (currentRegion != null) {
-          byte[] endKey = currentRegion.getEndKey();
-          if (endKey == null ||
-              Bytes.equals(endKey, HConstants.EMPTY_BYTE_ARRAY)) {
-            // We have reached the end of the table and we're done
-            break;
-          }
-        }
-        HRegionInfo oldRegion = currentRegion;
-        if (oldRegion != null) {
-          startKey = oldRegion.getEndKey();
-        }
-        s = new ScannerCallable(this, 
-            (Bytes.equals(tableName, HConstants.META_TABLE_NAME) ?
-                HConstants.ROOT_TABLE_NAME : HConstants.META_TABLE_NAME),
-            HConstants.COL_REGIONINFO_ARRAY, startKey,
-            HConstants.LATEST_TIMESTAMP, null
-        );
-        // Open scanner
-        getRegionServerWithRetries(s);
-        currentRegion = s.getHRegionInfo();
-        try {
-          RowResult r = null;
-          while ((r = getRegionServerWithRetries(s)) != null) {
-            Cell c = r.get(HConstants.COL_REGIONINFO);
-            if (c != null) {
-              HRegionInfo info = Writables.getHRegionInfoOrNull(c.getValue());
-              if (info != null) {
-                if (Bytes.equals(info.getTableDesc().getName(), tableName)) {
-                  result = new UnmodifyableHTableDescriptor(info.getTableDesc());
-                  break;
-                }
-              }
-            }
-          }
-        } finally {
-          s.setClose();
-          getRegionServerWithRetries(s);
-        }
+      HTableDescriptorFinder finder = new HTableDescriptorFinder(tableName);
+      MetaScanner.metaScan(conf, finder);
+      HTableDescriptor result = finder.getResult();
+      if (result == null) {
+        throw new TableNotFoundException(Bytes.toString(tableName));
       }
       return result;
     }

Modified: hadoop/hbase/branches/0.2/src/java/org/apache/hadoop/hbase/mapred/TableInputFormatBase.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.2/src/java/org/apache/hadoop/hbase/mapred/TableInputFormatBase.java?rev=684952&r1=684951&r2=684952&view=diff
==============================================================================
--- hadoop/hbase/branches/0.2/src/java/org/apache/hadoop/hbase/mapred/TableInputFormatBase.java
(original)
+++ hadoop/hbase/branches/0.2/src/java/org/apache/hadoop/hbase/mapred/TableInputFormatBase.java
Mon Aug 11 14:58:12 2008
@@ -26,6 +26,7 @@
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.UnknownScannerException;
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.client.Scanner;
 import org.apache.hadoop.hbase.filter.RowFilterInterface;
@@ -41,6 +42,7 @@
 import org.apache.hadoop.mapred.JobConf;
 import org.apache.hadoop.mapred.RecordReader;
 import org.apache.hadoop.mapred.Reporter;
+import org.apache.hadoop.util.StringUtils;
 
 /**
  * A Base for {@link TableInputFormat}s. Receives a {@link HTable}, a
@@ -85,17 +87,18 @@
   implements RecordReader<ImmutableBytesWritable, RowResult> {
     private byte [] startRow;
     private byte [] endRow;
+    private byte [] lastRow;
     private RowFilterInterface trrRowFilter;
     private Scanner scanner;
     private HTable htable;
     private byte [][] trrInputColumns;
 
     /**
-     * Build the scanner. Not done in constructor to allow for extension.
+     * Restart from survivable exceptions by creating a new scanner.
      *
      * @throws IOException
      */
-    public void init() throws IOException {
+    public void restart(byte[] firstRow) throws IOException {
       if ((endRow != null) && (endRow.length > 0)) {
         if (trrRowFilter != null) {
           final Set<RowFilterInterface> rowFiltersSet =
@@ -107,15 +110,24 @@
               rowFiltersSet));
         } else {
           this.scanner =
-            this.htable.getScanner(trrInputColumns, startRow, endRow);
+            this.htable.getScanner(trrInputColumns, firstRow, endRow);
         }
       } else {
         this.scanner =
-          this.htable.getScanner(trrInputColumns, startRow, trrRowFilter);
+          this.htable.getScanner(trrInputColumns, firstRow, trrRowFilter);
       }
     }
 
     /**
+     * Build the scanner. Not done in constructor to allow for extension.
+     *
+     * @throws IOException
+     */
+    public void init() throws IOException {
+      restart(startRow);
+    }
+
+    /**
      * @param htable the {@link HTable} to scan.
      */
     public void setHTable(HTable htable) {
@@ -190,19 +202,25 @@
     /**
      * @param key HStoreKey as input key.
      * @param value MapWritable as input value
-     *
-     * Converts Scanner.next() to Text, RowResult
-     *
      * @return true if there was more data
      * @throws IOException
      */
     @SuppressWarnings("unchecked")
     public boolean next(ImmutableBytesWritable key, RowResult value)
     throws IOException {
-      RowResult result = this.scanner.next();
+      RowResult result;
+      try {
+        result = this.scanner.next();
+      } catch (UnknownScannerException e) {
+        LOG.debug("recovered from " + StringUtils.stringifyException(e));  
+        restart(lastRow);
+        this.scanner.next();    // skip presumed already mapped row
+        result = this.scanner.next();
+      }
       boolean hasMore = result != null && result.size() > 0;
       if (hasMore) {
         key.set(result.getRow());
+        lastRow = key.get();
         Writables.copyWritable(result, value);
       }
       return hasMore;

Modified: hadoop/hbase/branches/0.2/src/test/org/apache/hadoop/hbase/client/TestHTable.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.2/src/test/org/apache/hadoop/hbase/client/TestHTable.java?rev=684952&r1=684951&r2=684952&view=diff
==============================================================================
--- hadoop/hbase/branches/0.2/src/test/org/apache/hadoop/hbase/client/TestHTable.java (original)
+++ hadoop/hbase/branches/0.2/src/test/org/apache/hadoop/hbase/client/TestHTable.java Mon
Aug 11 14:58:12 2008
@@ -150,36 +150,25 @@
       // enable the table
       admin.enableTable(tableAname);
 
-      // Use a metascanner to avoid client API caching (HConnection has a
-      // metadata cache)
-      MetaScanner.MetaScannerVisitor visitor =
-        new MetaScanner.MetaScannerVisitor() {
-          public boolean processRow(RowResult rowResult) throws IOException {
-            HRegionInfo info = Writables.getHRegionInfo(
-                rowResult.get(HConstants.COL_REGIONINFO));
+      // test that attribute changes were applied
+      desc = a.getTableDescriptor();
+      if (Bytes.compareTo(desc.getName(), tableAname) != 0)
+        fail("wrong table descriptor returned");
+      // check HTD attribute
+      value = desc.getValue(attrName);
+      if (value == null)
+        fail("missing HTD attribute value");
+      if (Bytes.compareTo(value, attrValue) != 0)
+        fail("HTD attribute value is incorrect");
+      // check HCD attribute
+      for (HColumnDescriptor c: desc.getFamilies()) {
+        value = c.getValue(attrName);
+        if (value == null)
+          fail("missing HCD attribute value");
+        if (Bytes.compareTo(value, attrValue) != 0)
+          fail("HCD attribute value is incorrect");
+      }
 
-            LOG.info("visiting " + info.toString());
-            HTableDescriptor desc = info.getTableDesc();
-            if (Bytes.compareTo(desc.getName(), tableAname) == 0) {
-              // check HTD attribute
-              byte[] value = desc.getValue(attrName);
-              if (value == null)
-                fail("missing HTD attribute value");
-              if (Bytes.compareTo(value, attrValue) != 0)
-                fail("HTD attribute value is incorrect");
-              // check HCD attribute
-              for (HColumnDescriptor c: desc.getFamilies()) {
-                value = c.getValue(attrName);
-                if (value == null)
-                  fail("missing HCD attribute value");
-                if (Bytes.compareTo(value, attrValue) != 0)
-                  fail("HCD attribute value is incorrect");
-              }
-            }
-            return true;
-          }
-        };
-        MetaScanner.metaScan(conf, visitor);
     } catch (Exception e) {
       e.printStackTrace();
       fail();



Mime
View raw message