hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From st...@apache.org
Subject svn commit: r614796 - in /hadoop/core/trunk/src/contrib/hbase: CHANGES.txt src/java/org/apache/hadoop/hbase/HMaster.java src/java/org/apache/hadoop/hbase/HStoreFile.java
Date Thu, 24 Jan 2008 05:13:27 GMT
Author: stack
Date: Wed Jan 23 21:13:22 2008
New Revision: 614796

URL: http://svn.apache.org/viewvc?rev=614796&view=rev
Log:
HADOOP-2688 IllegalArgumentException processing a shutdown stops server
going down and results in millions of lines of output

Modified:
    hadoop/core/trunk/src/contrib/hbase/CHANGES.txt
    hadoop/core/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HMaster.java
    hadoop/core/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HStoreFile.java

Modified: hadoop/core/trunk/src/contrib/hbase/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hbase/CHANGES.txt?rev=614796&r1=614795&r2=614796&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/hbase/CHANGES.txt (original)
+++ hadoop/core/trunk/src/contrib/hbase/CHANGES.txt Wed Jan 23 21:13:22 2008
@@ -146,6 +146,8 @@
    HADOOP-2668 Documentation and improved logging so fact that hbase now
                requires migration comes as less of a surprise
    HADOOP-2686 Removed tables stick around in .META.
+   HADOOP-2688 IllegalArgumentException processing a shutdown stops
+               server going down and results in millions of lines of output
    
   IMPROVEMENTS
    HADOOP-2401 Add convenience put method that takes writable

Modified: hadoop/core/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HMaster.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HMaster.java?rev=614796&r1=614795&r2=614796&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HMaster.java (original)
+++ hadoop/core/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HMaster.java Wed
Jan 23 21:13:22 2008
@@ -223,24 +223,19 @@
 
         int numberOfRegionsFound = 0;
         while (true) {
-          SortedMap<Text, byte[]> results = new TreeMap<Text, byte[]>();
           HbaseMapWritable values = regionServer.next(scannerId);
           if (values == null || values.size() == 0) {
             break;
           }
-          for (Map.Entry<Writable, Writable> e: values.entrySet()) {
-            HStoreKey key = (HStoreKey) e.getKey();
-            results.put(key.getColumn(),
-                ((ImmutableBytesWritable) e.getValue()).get());
-          }
-          byte [] bytes = results.get(COL_REGIONINFO);
-          if (bytes == null) {
-            LOG.warn(COL_REGIONINFO.toString() + " is empty; has keys: " +
-              values.keySet().toString());
+
+          // TODO: Why does this have to be a sorted map?
+          SortedMap<Text, byte[]> results = toRowMap(values).getMap();
+          
+          HRegionInfo info = getHRegionInfo(results);
+          if (info == null) {
             continue;
           }
-          HRegionInfo info = (HRegionInfo) Writables.getWritable(bytes,
-            new HRegionInfo());
+
           String serverName = Writables.bytesToString(results.get(COL_SERVER));
           long startCode = Writables.bytesToLong(results.get(COL_STARTCODE));
           if (LOG.isDebugEnabled()) {
@@ -2007,28 +2002,13 @@
               RemoteExceptionHandler.checkIOException(e));
             break;
           }
-
           if (values == null || values.size() == 0) {
             break;
           }
-
-          SortedMap<Text, byte[]> results = new TreeMap<Text, byte[]>();
-          Text row = null;
-          for (Map.Entry<Writable, Writable> e: values.entrySet()) {
-            HStoreKey key = (HStoreKey) e.getKey();
-            Text thisRow = key.getRow();
-            if (row == null) {
-              row = thisRow;
-            } else {
-              if (!row.equals(thisRow)) {
-                LOG.error("Multiple rows in same scanner result set. firstRow="
-                    + row + ", currentRow=" + thisRow);
-              }
-            }
-            results.put(key.getColumn(),
-                ((ImmutableBytesWritable) e.getValue()).get());
-          }
-
+          // TODO: Why does this have to be a sorted map?
+          RowMap rm = toRowMap(values);
+          Text row = rm.getRow();
+          SortedMap<Text, byte[]> map = rm.getMap();
           if (LOG.isDebugEnabled() && row != null) {
             LOG.debug("shutdown scanner looking at " + row.toString());
           }
@@ -2038,8 +2018,8 @@
           // missed edits in hlog because hdfs does not do write-append).
           String serverName;
           try {
-            serverName = Writables.bytesToString(results.get(COL_SERVER));
-          } catch(UnsupportedEncodingException e) {
+            serverName = Writables.bytesToString(map.get(COL_SERVER));
+          } catch (UnsupportedEncodingException e) {
             LOG.error("Server name", e);
             break;
           }
@@ -2054,13 +2034,9 @@
           }
 
           // Bingo! Found it.
-          HRegionInfo info = null;
-          try {
-            info = (HRegionInfo) Writables.getWritable(
-                results.get(COL_REGIONINFO), new HRegionInfo());
-          } catch (IOException e) {
-            LOG.error("Read fields", e);
-            break;
+          HRegionInfo info = getHRegionInfo(map);
+          if (info == null) {
+            continue;
           }
           LOG.info(info.getRegionName() + " was on shutdown server <" +
               serverName + "> (or server is null). Marking unassigned in " +
@@ -2720,52 +2696,21 @@
 
               try {
                 while (true) {
-                  HRegionInfo info = new HRegionInfo();
-                  String serverName = null;
-                  long startCode = -1L;
-
                   HbaseMapWritable values = server.next(scannerId);
                   if(values == null || values.size() == 0) {
                     break;
                   }
-                  boolean haveRegionInfo = false;
-                  for (Map.Entry<Writable, Writable> e: values.entrySet()) {
-
-                    byte[] value = ((ImmutableBytesWritable) e.getValue()).get();
-                    if (value == null || value.length == 0) {
-                      break;
-                    }
-                    HStoreKey key = (HStoreKey) e.getKey();
-                    Text column = key.getColumn();
-                    if (column.equals(COL_REGIONINFO)) {
-                      haveRegionInfo = true;
-                      info = (HRegionInfo) Writables.getWritable(value, info);
-                    
-                    } else if (column.equals(COL_SERVER)) {
-                      try {
-                        serverName =
-                          Writables.bytesToString(value);
-                    
-                      } catch (UnsupportedEncodingException ex) {
-                        assert(false);
-                      }
-                    
-                    } else if (column.equals(COL_STARTCODE)) {
-                      try {
-                        startCode = Writables.bytesToLong(value);
-                      
-                      } catch (UnsupportedEncodingException ex) {
-                        assert(false);
-                      }
-                    }
-                  }
-
-                  if (!haveRegionInfo) {
-                    throw new IOException(COL_REGIONINFO + " not found");
+                  RowMap rm = toRowMap(values);
+                  SortedMap<Text, byte[]> map = rm.getMap();
+                  HRegionInfo info = getHRegionInfo(map);
+                  if (info == null) {
+                    throw new IOException(COL_REGIONINFO + " not found on " +
+                      rm.getRow());
                   }
-
+                  String serverName = Writables.bytesToString(map.get(COL_SERVER));
+                  long startCode = Writables.bytesToLong(map.get(COL_STARTCODE));
                   if (info.getTableDesc().getName().compareTo(tableName) > 0) {
-                    break;               // Beyond any more entries for this table
+                    break; // Beyond any more entries for this table
                   }
 
                   tableExists = true;
@@ -2773,9 +2718,7 @@
                     unservedRegions.add(info);
                   }
                   processScanItem(serverName, startCode, info);
-
                 } // while(true)
-
               } finally {
                 if (scannerId != -1L) {
                   try {
@@ -2994,7 +2937,7 @@
     protected void updateRegionInfo(BatchUpdate b,
         @SuppressWarnings("unused") HRegionInfo info) {
       for (int i = 0; i < ALL_META_COLUMNS.length; i++) {
-        // Be sure to clean all columns
+        // Be sure to clean all cells
         b.delete(lockid, ALL_META_COLUMNS[i]);
       }
     }
@@ -3169,6 +3112,75 @@
    */
   public HBaseConfiguration getConfiguration() {
     return this.conf;
+  }
+  
+  /*
+   * Data structure used to return results out of the toRowMap method.
+   */
+  private class RowMap {
+    final Text row;
+    final SortedMap<Text, byte[]> map;
+    
+    private RowMap(final Text r, final SortedMap<Text, byte[]> m) {
+      this.row = r;
+      this.map = m;
+    }
+
+    private Text getRow() {
+      return this.row;
+    }
+
+    private SortedMap<Text, byte[]> getMap() {
+      return this.map;
+    }
+  }
+  
+  /*
+   * Convert an HbaseMapWritable to a Map keyed by column.
+   * Utility method used scanning meta regions
+   * @param mw The MapWritable to convert.  Cannot be null.
+   * @return Returns a SortedMap currently.  TODO: This looks like it could
+   * be a plain Map.
+   */
+  protected RowMap toRowMap(final HbaseMapWritable mw) {
+    if (mw == null) {
+      throw new IllegalArgumentException("Passed MapWritable cannot be null");
+    }
+    SortedMap<Text, byte[]> m = new TreeMap<Text, byte[]>();
+    Text row = null;
+    for (Map.Entry<Writable, Writable> e: mw.entrySet()) {
+      HStoreKey key = (HStoreKey) e.getKey();
+      Text thisRow = key.getRow();
+      if (row == null) {
+        row = thisRow;
+      } else {
+        if (!row.equals(thisRow)) {
+          LOG.error("Multiple rows in same scanner result set. firstRow=" +
+            row + ", currentRow=" + thisRow);
+        }
+      }
+      m.put(key.getColumn(), ((ImmutableBytesWritable) e.getValue()).get());
+    }
+    return new RowMap(row, m);
+  }
+  
+  /*
+   * Get HRegionInfo from passed META map of row values.
+   * Returns null if none found (and logs fact that expected COL_REGIONINFO
+   * was missing).  Utility method used by scanners of META tables.
+   * @param map Map to do lookup in.
+   * @return Null or found HRegionInfo.
+   * @throws IOException
+   */
+  protected HRegionInfo getHRegionInfo(final Map<Text, byte[]> map)
+  throws IOException {
+    byte [] bytes = map.get(COL_REGIONINFO);
+    if (bytes == null) {
+      LOG.warn(COL_REGIONINFO.toString() + " is empty; has keys: " +
+        map.keySet().toString());
+      return null;
+    }
+    return (HRegionInfo)Writables.getWritable(bytes, new HRegionInfo());
   }
 
   /*

Modified: hadoop/core/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HStoreFile.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HStoreFile.java?rev=614796&r1=614795&r2=614796&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HStoreFile.java (original)
+++ hadoop/core/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HStoreFile.java Wed
Jan 23 21:13:22 2008
@@ -413,15 +413,13 @@
    * @return MapFile.Reader
    * @throws IOException
    */
-  public synchronized MapFile.Reader getReader(final FileSystem fs,
-      final Filter bloomFilter) throws IOException {
-    
-    if (isReference()) {
-      return new HStoreFile.HalfMapFileReader(fs,
-          getMapFilePath(reference).toString(), conf, 
-          reference.getFileRegion(), reference.getMidkey(), bloomFilter);
-    }
-    return new BloomFilterMapFile.Reader(fs, getMapFilePath().toString(),
+  public MapFile.Reader getReader(final FileSystem fs,
+    final Filter bloomFilter)
+  throws IOException {
+    return isReference()?
+      new HStoreFile.HalfMapFileReader(fs, getMapFilePath(reference).toString(),
+        conf, reference.getFileRegion(), reference.getMidkey(), bloomFilter):
+      new BloomFilterMapFile.Reader(fs, getMapFilePath().toString(),
         conf, bloomFilter);
   }
 



Mime
View raw message