hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From st...@apache.org
Subject svn commit: r787857 - in /hadoop/hbase/branches/0.19: ./ src/java/org/apache/hadoop/hbase/ src/java/org/apache/hadoop/hbase/master/ src/java/org/apache/hadoop/hbase/regionserver/ src/test/org/apache/hadoop/hbase/io/ src/test/org/apache/hadoop/hbase/reg...
Date Tue, 23 Jun 2009 22:34:05 GMT
Author: stack
Date: Tue Jun 23 22:34:05 2009
New Revision: 787857

URL: http://svn.apache.org/viewvc?rev=787857&view=rev
Log:
HBASE-1572 Holes in master state change; updated startcode and server go into .META. but catalog
scanner just got old values

Modified:
    hadoop/hbase/branches/0.19/CHANGES.txt
    hadoop/hbase/branches/0.19/src/java/org/apache/hadoop/hbase/HTableDescriptor.java
    hadoop/hbase/branches/0.19/src/java/org/apache/hadoop/hbase/master/BaseScanner.java
    hadoop/hbase/branches/0.19/src/java/org/apache/hadoop/hbase/master/HMaster.java
    hadoop/hbase/branches/0.19/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java
    hadoop/hbase/branches/0.19/src/test/org/apache/hadoop/hbase/io/TestHbaseObjectWritable.java
    hadoop/hbase/branches/0.19/src/test/org/apache/hadoop/hbase/regionserver/TestScanner.java

Modified: hadoop/hbase/branches/0.19/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.19/CHANGES.txt?rev=787857&r1=787856&r2=787857&view=diff
==============================================================================
--- hadoop/hbase/branches/0.19/CHANGES.txt (original)
+++ hadoop/hbase/branches/0.19/CHANGES.txt Tue Jun 23 22:34:05 2009
@@ -15,6 +15,8 @@
    HBASE-1524  TableInputFormatBase.TableRecordReader.restart() @line 110.
                "startRow" should be changed to "firstRow"
                (Qiaoilang Xiang via Stack)
+   HBASE-1573  Holes in master state change; updated startcode and server go
+               into .META. but catalog scanner just got old values
 
 Release 0.19.3 - May 27th, 2009
   BUG FIXES

Modified: hadoop/hbase/branches/0.19/src/java/org/apache/hadoop/hbase/HTableDescriptor.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.19/src/java/org/apache/hadoop/hbase/HTableDescriptor.java?rev=787857&r1=787856&r2=787857&view=diff
==============================================================================
--- hadoop/hbase/branches/0.19/src/java/org/apache/hadoop/hbase/HTableDescriptor.java (original)
+++ hadoop/hbase/branches/0.19/src/java/org/apache/hadoop/hbase/HTableDescriptor.java Tue
Jun 23 22:34:05 2009
@@ -112,7 +112,7 @@
       this.families.put(Bytes.mapKey(descriptor.getName()), descriptor);
     }
     // TODO: Fix this hard-coding.
-    setMemcacheFlushSize(16 * 1024);
+    setMemcacheFlushSize(32 * 1024);
   }
 
   /**
@@ -675,7 +675,7 @@
   public static final HTableDescriptor ROOT_TABLEDESC = new HTableDescriptor(
       HConstants.ROOT_TABLE_NAME,
       new HColumnDescriptor[] { new HColumnDescriptor(HConstants.COLUMN_FAMILY,
-          10,  // Ten is arbitrary number.  Keep versions to help debuggging.
+          10,  // Ten is arbitrary number.  Keep versions to help debugging.
           HColumnDescriptor.CompressionType.NONE, false, true,
           Integer.MAX_VALUE, HConstants.FOREVER, false) });
   
@@ -683,7 +683,7 @@
   public static final HTableDescriptor META_TABLEDESC = new HTableDescriptor(
       HConstants.META_TABLE_NAME, new HColumnDescriptor[] {
           new HColumnDescriptor(HConstants.COLUMN_FAMILY,
-            10, // Ten is arbitrary number.  Keep versions to help debuggging.
+            10, // Ten is arbitrary number.  Keep versions to help debugging.
             HColumnDescriptor.CompressionType.NONE, false, true,
             Integer.MAX_VALUE, HConstants.FOREVER, false),
           new HColumnDescriptor(HConstants.COLUMN_FAMILY_HISTORIAN,

Modified: hadoop/hbase/branches/0.19/src/java/org/apache/hadoop/hbase/master/BaseScanner.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.19/src/java/org/apache/hadoop/hbase/master/BaseScanner.java?rev=787857&r1=787856&r2=787857&view=diff
==============================================================================
--- hadoop/hbase/branches/0.19/src/java/org/apache/hadoop/hbase/master/BaseScanner.java (original)
+++ hadoop/hbase/branches/0.19/src/java/org/apache/hadoop/hbase/master/BaseScanner.java Tue
Jun 23 22:34:05 2009
@@ -43,6 +43,7 @@
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.HColumnDescriptor;
 import org.apache.hadoop.hbase.io.BatchUpdate;
+import org.apache.hadoop.hbase.io.Cell;
 import org.apache.hadoop.hbase.io.RowResult;
 
 import org.apache.hadoop.hbase.regionserver.HRegion;
@@ -171,7 +172,7 @@
         long startCode = Writables.cellToLong(values.get(COL_STARTCODE));
 
         // Note Region has been assigned.
-        checkAssigned(info, serverName, startCode);
+        checkAssigned(info, serverName, startCode, region, values.getRow());
         if (isSplitParent(info)) {
           splitParents.put(info, values);
         }
@@ -332,7 +333,8 @@
   }
 
   protected void checkAssigned(final HRegionInfo info,
-    final String serverName, final long startCode) 
+    final String serverName, final long startCode, final MetaRegion metaregion,
+    final byte [] row) 
   throws IOException {
      HServerInfo storedInfo = null;
     synchronized (this.master.regionManager) {
@@ -342,22 +344,34 @@
        * by ProcessServerShutdown
        */
       if (info.isOffline() ||
-          this.master.regionManager.
-            regionIsInTransition(info.getRegionName()) ||
+          this.master.regionManager.regionIsInTransition(info.getRegionName()) ||
           (serverName != null && this.master.serverManager.isDead(serverName))) {
 
         return;
       }
       if (serverName != null && serverName.length() != 0) {
         storedInfo = this.master.serverManager.getServerInfo(serverName);
+        if (storedInfo == null) {
+          // Maybe its been updated since we do next on scan?  Check.
+          // We're doing this because race conditions in master -- could have
+          // had my serverinfo state updated and .META. updated but the current
+          // next may not see it.  Need to do this to avoid double assignment.
+          String newServerName = regetServerName(metaregion, row);
+          if (newServerName != null && !newServerName.equals(serverName)) {
+            // Got a different name.  Retry get of serverinfo.
+            LOG.debug("Got new server name on retry: " + newServerName + ", " +
+              serverName);
+            storedInfo = this.master.serverManager.getServerInfo(serverName);
+          }
+        }
       }
 
       /*
        * If the startcode is off -- either null or doesn't match the start code
        * for the address -- then add it to the list of unassigned regions.
        */ 
-      if (storedInfo == null || storedInfo.getStartCode() != startCode) {
-
+      if (storedInfo == null || (storedInfo.getStartCode() != startCode &&
+          regetStartCode(storedInfo.getStartCode(), metaregion, row))) {
         // The current assignment is invalid
         if (LOG.isDebugEnabled()) {
           LOG.debug("Current assignment of " + info.getRegionNameAsString() +
@@ -373,7 +387,7 @@
         // data in the meta region. Once we are on-line, dead server log
         // recovery is handled by lease expiration and ProcessServerShutdown
         if (!this.master.regionManager.isInitialMetaScanComplete() &&
-            serverName.length() != 0) {
+            (serverName != null && serverName.length() != 0)) {
           StringBuilder dirName = new StringBuilder("log_");
           dirName.append(serverName.replace(":", "_"));
           Path logDir = new Path(this.master.rootdir, dirName.toString());
@@ -401,6 +415,49 @@
     }
   }
 
+  /*
+   * @param metaregion
+   * @param row
+   * @return Server name or null if none found.
+   * @throws IOException
+   */
+  private String regetServerName(final MetaRegion metaregion, final byte [] row)
+  throws IOException {
+    HRegionInterface server = this.master.connection.
+    getHRegionConnection(metaregion.getServer());
+    Cell [] cells = server.get(metaregion.getRegionName(), row,
+        HConstants.COL_SERVER, HConstants.LATEST_TIMESTAMP, 1);
+    return cells.length > 0? Bytes.toString(cells[0].getValue()): "";
+  }
+
+  /*
+   * Go get the startcode from .META. again.  It may have changed since we
+   * started up the scan, especially if scan was big.
+   * @param currentStartCode What we have as current start code.
+   * @param metaregion Region to reget from.
+   * @param row Row in metaregion to get startcode from.
+   * @return True if regotten startcode does not match what we have for
+   * server info or if we fail to get a startcode.
+   * @throws IOException 
+   */
+  private boolean regetStartCode(final long currentStartCode,
+      final MetaRegion metaregion, final byte [] row)
+  throws IOException {
+    HRegionInterface server = this.master.connection.
+      getHRegionConnection(metaregion.getServer());
+    Cell [] cells = server.get(metaregion.getRegionName(), row,
+      HConstants.COL_STARTCODE, HConstants.LATEST_TIMESTAMP, 1);
+    if (cells.length > 0) {
+      long newStartCode = Bytes.toLong(cells[0].getValue());
+      boolean b = newStartCode != currentStartCode;
+      if (!b) {
+        LOG.debug("Reget found a different start code: " +
+          currentStartCode + ", " + newStartCode);
+      }
+    }
+    return true;
+  }
+
   /**
    * Notify the thread to die at the end of its next run
    */
@@ -411,4 +468,4 @@
       }
     }
   }
-}
+}
\ No newline at end of file

Modified: hadoop/hbase/branches/0.19/src/java/org/apache/hadoop/hbase/master/HMaster.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.19/src/java/org/apache/hadoop/hbase/master/HMaster.java?rev=787857&r1=787856&r2=787857&view=diff
==============================================================================
--- hadoop/hbase/branches/0.19/src/java/org/apache/hadoop/hbase/master/HMaster.java (original)
+++ hadoop/hbase/branches/0.19/src/java/org/apache/hadoop/hbase/master/HMaster.java Tue Jun
23 22:34:05 2009
@@ -30,9 +30,8 @@
 import java.util.Set;
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.DelayQueue;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.TimeUnit;
 import java.util.concurrent.PriorityBlockingQueue;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 
 import org.apache.commons.logging.Log;
@@ -95,8 +94,7 @@
   
   static final Log LOG = LogFactory.getLog(HMaster.class.getName());
 
-  public long getProtocolVersion(@SuppressWarnings("unused") String protocol,
-      @SuppressWarnings("unused") long clientVersion) {
+  public long getProtocolVersion(final String protocol, final long clientVersion) {
     return HBaseRPCProtocolVersion.versionID;
   }
 

Modified: hadoop/hbase/branches/0.19/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.19/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java?rev=787857&r1=787856&r2=787857&view=diff
==============================================================================
--- hadoop/hbase/branches/0.19/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java
(original)
+++ hadoop/hbase/branches/0.19/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java
Tue Jun 23 22:34:05 2009
@@ -44,7 +44,6 @@
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.ColumnNameParseException;
-import org.apache.hadoop.hbase.DoNotRetryIOException;
 import org.apache.hadoop.hbase.DroppedSnapshotException;
 import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.HColumnDescriptor;

Modified: hadoop/hbase/branches/0.19/src/test/org/apache/hadoop/hbase/io/TestHbaseObjectWritable.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.19/src/test/org/apache/hadoop/hbase/io/TestHbaseObjectWritable.java?rev=787857&r1=787856&r2=787857&view=diff
==============================================================================
--- hadoop/hbase/branches/0.19/src/test/org/apache/hadoop/hbase/io/TestHbaseObjectWritable.java
(original)
+++ hadoop/hbase/branches/0.19/src/test/org/apache/hadoop/hbase/io/TestHbaseObjectWritable.java
Tue Jun 23 22:34:05 2009
@@ -77,6 +77,7 @@
     obj = doType(conf, new StopRowFilter(HConstants.EMPTY_BYTE_ARRAY),
         RowFilterInterface.class);
     assertTrue(obj instanceof StopRowFilter);
+    // Try new Filter type.
   }
   
   private Object doType(final HBaseConfiguration conf, final Object value,
@@ -93,4 +94,4 @@
     dis.close();
     return product;
   }
-}
+}
\ No newline at end of file

Modified: hadoop/hbase/branches/0.19/src/test/org/apache/hadoop/hbase/regionserver/TestScanner.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.19/src/test/org/apache/hadoop/hbase/regionserver/TestScanner.java?rev=787857&r1=787856&r2=787857&view=diff
==============================================================================
--- hadoop/hbase/branches/0.19/src/test/org/apache/hadoop/hbase/regionserver/TestScanner.java
(original)
+++ hadoop/hbase/branches/0.19/src/test/org/apache/hadoop/hbase/regionserver/TestScanner.java
Tue Jun 23 22:34:05 2009
@@ -72,6 +72,11 @@
     // Set the hbase.rootdir to be the home directory in mini dfs.
     this.conf.set(HConstants.HBASE_DIR,
       this.cluster.getFileSystem().getHomeDirectory().toString());
+    // There is no flusher thread running to notice when the 'blocker' comes
+    // down when we are over memcache size limits. Therefore stop 'blocker'
+    // from working in this test.  Otherwise was seeing these:
+    // 2009-06-23 15:07:13,876 INFO  [main] regionserver.HRegion(1412): Blocking updates
for 'main' on region -ROOT-,,1245794831727: Memcache size 3.1m is >= than blocking 3.1m
size
+    this.conf.setInt("hbase.hregion.memcache.block.multiplier", 1000);
     super.setUp();
     
   }



Mime
View raw message