hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From apurt...@apache.org
Subject svn commit: r721411 - in /hadoop/hbase/branches/0.19_on_hadoop_0.18: ./ src/java/org/apache/hadoop/hbase/ src/java/org/apache/hadoop/hbase/client/ src/java/org/apache/hadoop/hbase/ipc/ src/java/org/apache/hadoop/hbase/master/ src/java/org/apache/hadoop...
Date Fri, 28 Nov 2008 10:14:23 GMT
Author: apurtell
Date: Fri Nov 28 02:14:21 2008
New Revision: 721411

URL: http://svn.apache.org/viewvc?rev=721411&view=rev
Log:
merge up to trunk (revision 721410)

Modified:
    hadoop/hbase/branches/0.19_on_hadoop_0.18/CHANGES.txt
    hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/HServerLoad.java
    hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
    hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/ipc/HBaseRPCProtocolVersion.java
    hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/master/RegionManager.java
    hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java
    hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
    hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/regionserver/HStore.java

Modified: hadoop/hbase/branches/0.19_on_hadoop_0.18/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.19_on_hadoop_0.18/CHANGES.txt?rev=721411&r1=721410&r2=721411&view=diff
==============================================================================
--- hadoop/hbase/branches/0.19_on_hadoop_0.18/CHANGES.txt (original)
+++ hadoop/hbase/branches/0.19_on_hadoop_0.18/CHANGES.txt Fri Nov 28 02:14:21 2008
@@ -11,6 +11,8 @@
                caching work....?] -- Update your hbad-default.xml file!
    HBASE-636   java6 as a requirement
    HBASE-994   IPC interfaces with different versions can cause problems
+   HBASE-1028  If key does not exist, return null in getRow rather than an
+               empty RowResult
 
   BUG FIXES
    HBASE-891   HRS.validateValuesLength throws IOE, gets caught in the retries
@@ -140,7 +142,9 @@
    HBASE-972   Update hbase trunk to use released hadoop 0.19.0
    HBASE-1022  Add storefile index size to hbase metrics
    HBASE-1026  Tests in mapred are failing
- 
+   HBASE-1020  Regionserver OOME handler should dump vital stats
+   HBASE-1018  Regionservers should report detailed health to master
+
   NEW FEATURES
    HBASE-875   Use MurmurHash instead of JenkinsHash [in bloomfilters]
                (Andrzej Bialecki via Stack)

Modified: hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/HServerLoad.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/HServerLoad.java?rev=721411&r1=721410&r2=721411&view=diff
==============================================================================
--- hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/HServerLoad.java
(original)
+++ hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/HServerLoad.java
Fri Nov 28 02:14:21 2008
@@ -22,25 +22,171 @@
 import java.io.DataInput;
 import java.io.DataOutput;
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Iterator;
 
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.io.Writable;
 import org.apache.hadoop.io.WritableComparable;
 
 /**
  * This class encapsulates metrics for determining the load on a HRegionServer
  */
 public class HServerLoad implements WritableComparable {
-  private int numberOfRequests;         // number of requests since last report
-  private int numberOfRegions;          // number of regions being served
-  /*
-   * Number of storefiles on the regionserver
-   */
-  private int storefiles;
-  
-  /*
-   * Size of the memcaches on this machine in MB.
-   */
-  private int memcacheSizeMB;
-  
+  /** number of regions */
+    // could just use regionLoad.size() but master.RegionManager likes to play
+    // around with this value while passing HServerLoad objects around during
+    // balancer calculations
+  private int numberOfRegions;
+  /** number of requests since last report */
+  private int numberOfRequests;
+  /** the amount of used heap, in MB */
+  private int usedHeapMB;
+  /** the maximum allowable size of the heap, in MB */
+  private int maxHeapMB;
+  /** per-region load metrics */
+  private ArrayList<RegionLoad> regionLoad = new ArrayList<RegionLoad>();
+
+  /** 
+   * Encapsulates per-region loading metrics.
+   */
+  class RegionLoad implements Writable {
+    /** the region name */
+    private byte[] name;
+    /** the number of stores for the region */
+    private int stores;
+    /** the number of storefiles for the region */
+    private int storefiles;
+    /** the current size of the memcache for the region, in MB */
+    private int memcacheSizeMB;
+    /** the current total size of storefile indexes for the region, in MB */
+    private int storefileIndexSizeMB;
+
+    /**
+     * Constructor, for Writable
+     */
+    public RegionLoad() {
+        super();
+    }
+
+    /**
+     * @param name
+     * @param stores
+     * @param storefiles
+     * @param memcacheSizeMB
+     * @param storefileIndexSizeMB
+     */
+    public RegionLoad(final byte[] name, final int stores,
+        final int storefiles, final int memcacheSizeMB,
+        final int storefileIndexSizeMB) {
+      this.name = name;
+      this.stores = stores;
+      this.storefiles = storefiles;
+      this.memcacheSizeMB = memcacheSizeMB;
+      this.storefileIndexSizeMB = storefileIndexSizeMB;
+    }
+
+    // Getters
+
+    /**
+     * @return the region name
+     */
+    public byte[] getName() {
+      return name;
+    }
+
+    /**
+     * @return the number of stores
+     */
+    public int getStores() {
+      return stores;
+    }
+
+    /**
+     * @return the number of storefiles
+     */
+    public int getStorefiles() {
+      return storefiles;
+    }
+
+    /**
+     * @return the memcache size, in MB
+     */
+    public int getMemcacheSizeMB() {
+      return memcacheSizeMB;
+    }
+
+    /**
+     * @return the approximate size of storefile indexes on the heap, in MB
+     */
+    public int getStorefileIndexSizeMB() {
+      return storefileIndexSizeMB;
+    }
+
+    // Setters
+
+    /**
+     * @param name the region name
+     */
+    public void setName(byte[] name) {
+      this.name = name;
+    }
+
+    /**
+     * @param storefiles the number of stores
+     */
+    public void setStores(int stores) {
+      this.stores = stores;
+    }
+
+    /**
+     * @param storefiles the number of storefiles
+     */
+    public void setStorefiles(int storefiles) {
+      this.storefiles = storefiles;
+    }
+
+    /**
+     * @param memcacheSizeMB the memcache size, in MB
+     */
+    public void setMemcacheSizeMB(int memcacheSizeMB) {
+      this.memcacheSizeMB = memcacheSizeMB;
+    }
+
+    /**
+     * @param storefileIndexSizeMB the approximate size of storefile indexes
+     *  on the heap, in MB
+     */
+    public void setStorefileIndexSizeMB(int storefileIndexSizeMB) {
+      this.storefileIndexSizeMB = storefileIndexSizeMB;
+    }
+
+    // Writable
+
+    @Override
+    public void readFields(DataInput in) throws IOException {
+      int namelen = in.readInt();
+      this.name = new byte[namelen];
+      in.readFully(this.name);
+      this.stores = in.readInt();
+      this.storefiles = in.readInt();
+      this.memcacheSizeMB = in.readInt();
+      this.storefileIndexSizeMB = in.readInt();
+    }
+
+    @Override
+    public void write(DataOutput out) throws IOException {
+      out.writeInt(name.length);
+      out.write(name);
+      out.writeInt(stores);
+      out.writeInt(storefiles);
+      out.writeInt(memcacheSizeMB);
+      out.writeInt(storefileIndexSizeMB);
+    }
+  }
+
   /*
    * TODO: Other metrics that might be considered when the master is actually
    * doing load balancing instead of merely trying to decide where to assign
@@ -54,7 +200,7 @@
    * </ul>
    */
 
-  /** default constructior (used by Writable) */
+  /** default constructor (used by Writable) */
   public HServerLoad() {
     super();
   }
@@ -62,21 +208,25 @@
   /**
    * Constructor
    * @param numberOfRequests
-   * @param numberOfRegions
+   * @param usedHeapMB
+   * @param maxHeapMB
    */
-  public HServerLoad(final int numberOfRequests, final int numberOfRegions,
-      final int storefiles, final int memcacheSizeMB) {
+  public HServerLoad(final int numberOfRequests, final int usedHeapMB,
+      final int maxHeapMB) {
     this.numberOfRequests = numberOfRequests;
-    this.numberOfRegions = numberOfRegions;
-    this.storefiles = storefiles;
-    this.memcacheSizeMB = memcacheSizeMB;
+    this.usedHeapMB = usedHeapMB;
+    this.maxHeapMB = maxHeapMB;
   }
-  
+
+  /**
+   * Constructor
+   * @param hsl the template HServerLoad
+   */
   public HServerLoad(final HServerLoad hsl) {
-    this(hsl.numberOfRequests, hsl.numberOfRegions, hsl.storefiles,
-      hsl.memcacheSizeMB);
+    this(hsl.numberOfRequests, hsl.usedHeapMB, hsl.maxHeapMB);
+    this.regionLoad.addAll(hsl.regionLoad);
   }
-  
+
   /**
    * Originally, this method factored in the effect of requests going to the
    * server as well. However, this does not interact very well with the current
@@ -99,15 +249,43 @@
   }
   
   /**
-   * Returns toString() with the number of requests divided by the message interval in seconds
+   * Returns toString() with the number of requests divided by the message
+   * interval in seconds
    * @param msgInterval
    * @return The load as a String
    */
   public String toString(int msgInterval) {
-    return "requests: " + numberOfRequests/msgInterval +
-      " regions: " + numberOfRegions;
+    StringBuilder sb = new StringBuilder();
+    sb.append("requests: ");
+    sb.append(numberOfRequests/msgInterval);
+    sb.append(" usedHeapMB: ");
+    sb.append(usedHeapMB);
+    sb.append(" maxHeapMB: ");
+    sb.append(maxHeapMB);
+    sb.append(" regions: ");
+    sb.append(numberOfRegions);
+    Iterator<RegionLoad> i = regionLoad.iterator();
+    sb.append(" {");
+    while (i.hasNext()) {
+        RegionLoad rl = i.next();
+        sb.append(" { name: '");
+        sb.append(Bytes.toString(rl.name));
+        sb.append("' stores: ");
+        sb.append(rl.stores);
+        sb.append(" storefiles: ");
+        sb.append(rl.storefiles);
+        sb.append(" memcacheSizeMB: ");
+        sb.append(rl.memcacheSizeMB);
+        sb.append(" storefileIndexSizeMB: ");
+        sb.append(rl.storefileIndexSizeMB);
+        sb.append(" }");
+        if (i.hasNext())
+            sb.append(',');
+    }
+    sb.append(" }");
+    return sb.toString();
   }
-  
+
   @Override
   public boolean equals(Object o) {
     return compareTo(o) == 0;
@@ -129,6 +307,10 @@
     return numberOfRegions;
   }
 
+  public Collection<RegionLoad> getRegionLoad() {
+    return Collections.unmodifiableCollection(regionLoad);
+  }
+
   /**
    * @return the numberOfRequests
    */
@@ -140,56 +322,101 @@
    * @return Count of storefiles on this regionserver
    */
   public int getStorefiles() {
-    return this.storefiles;
+    int count = 0;
+    for (RegionLoad info: regionLoad)
+      count += info.storefiles;
+    return count;
   }
 
   /**
-   * @return Size of memcaches in kb.
+   * @return Size of memcaches in MB
    */
-  public int getMemcacheSizeInKB() {
-    return this.memcacheSizeMB;
+  public int getMemcacheSizeInMB() {
+    int count = 0;
+    for (RegionLoad info: regionLoad)
+      count += info.memcacheSizeMB;
+    return count;
   }
 
   /**
-   * @param storefiles Count of storefiles on this server.
+   * @return Size of store file indexes in MB
    */
-  public void setStorefiles(int storefiles) {
-    this.storefiles = storefiles;
+  public int getStorefileIndexSizeInMB() {
+    int count = 0;
+    for (RegionLoad info: regionLoad)
+      count += info.storefileIndexSizeMB;
+    return count;
   }
 
-  /**
-   * @param memcacheSizeInKB Size of memcache in kb.
-   */
-  public void setMemcacheSizeInKB(int memcacheSizeInKB) {
-    this.memcacheSizeMB = memcacheSizeInKB;
-  }
+  // Setters
 
   /**
-   * @param numberOfRegions the numberOfRegions to set
+   * @param numberOfRegions the number of regions
    */
   public void setNumberOfRegions(int numberOfRegions) {
     this.numberOfRegions = numberOfRegions;
   }
 
   /**
-   * @param numberOfRequests the numberOfRequests to set
+   * @param numberOfRequests the number of requests to set
    */
   public void setNumberOfRequests(int numberOfRequests) {
     this.numberOfRequests = numberOfRequests;
   }
 
+  /**
+   * @param usedHeapMB the amount of heap in use, in MB
+   */
+  public void setUsedHeapMB(int usedHeapMB) {
+    this.usedHeapMB = usedHeapMB;
+  }
+
+  /**
+   * @param maxHeapMB the maximum allowable heap size, in MB
+   */
+  public void setMaxHeapMB(int maxHeapMB) {
+    this.maxHeapMB = maxHeapMB;
+  }
+
+  /**
+   * @param name
+   * @param stores
+   * @param storefiles
+   * @param memcacheSizeMB
+   * @param storefileIndexSizeMB
+   */
+  public void addRegionInfo(final byte[] name, final int stores,
+      final int storefiles, final int memcacheSizeMB,
+      final int storefileIndexSizeMB) {
+    this.numberOfRegions++;
+    this.regionLoad.add(
+      new RegionLoad(name, stores, storefiles, memcacheSizeMB,
+        storefileIndexSizeMB));
+  }
+
   // Writable
 
   public void readFields(DataInput in) throws IOException {
     numberOfRequests = in.readInt();
+    usedHeapMB = in.readInt();
+    maxHeapMB = in.readInt();
     numberOfRegions = in.readInt();
+    for (int i = 0; i < numberOfRegions; i++) {
+      RegionLoad rl = new RegionLoad();
+      rl.readFields(in);
+      regionLoad.add(rl);
+    }
   }
 
   public void write(DataOutput out) throws IOException {
     out.writeInt(numberOfRequests);
+    out.writeInt(usedHeapMB);
+    out.writeInt(maxHeapMB);
     out.writeInt(numberOfRegions);
+    for (int i = 0; i < numberOfRegions; i++)
+      regionLoad.get(i).write(out);
   }
-  
+
   // Comparable
 
   public int compareTo(Object o) {

Modified: hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/client/HBaseAdmin.java?rev=721411&r1=721410&r2=721411&view=diff
==============================================================================
--- hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
(original)
+++ hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
Fri Nov 28 02:14:21 2008
@@ -529,6 +529,7 @@
         arr[0] = (HTableDescriptor)args[0];
         this.master.modifyTable(tableName, op, arr);
       } break;
+      case HConstants.MODIFY_TABLE_COMPACT:
       case HConstants.MODIFY_TABLE_SPLIT: {
         Writable[] arr = null;
         if (args != null && args.length > 0) {
@@ -538,8 +539,7 @@
           } else if (args[0] instanceof ImmutableBytesWritable) {
             arr[0] = (ImmutableBytesWritable)args[0];
           } else {
-            throw new IOException(
-              "SPLIT with arg requires byte[] or ImmutableBytesWritable");
+            throw new IOException("SPLIT or COMPACT with arg requires byte[] or ImmutableBytesWritable");
           }
         }
         this.master.modifyTable(tableName, op, arr);

Modified: hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/ipc/HBaseRPCProtocolVersion.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/ipc/HBaseRPCProtocolVersion.java?rev=721411&r1=721410&r2=721411&view=diff
==============================================================================
--- hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/ipc/HBaseRPCProtocolVersion.java
(original)
+++ hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/ipc/HBaseRPCProtocolVersion.java
Fri Nov 28 02:14:21 2008
@@ -60,9 +60,10 @@
    * <p>Unified RPC version number history:
    * <ul>
    * <li>Version 10: initial version (had to be &gt all other RPC versions</li>
-   * <li>Version 11: Changed getClosestRowBefore signature.
+   * <li>Version 11: Changed getClosestRowBefore signature.</li>
+   * <li>Version 12: HServerLoad extensions (HBASE-1018).</li>
    * </ul>
    */
-  public static final long versionID = 11L;
+  public static final long versionID = 12L;
 
 }

Modified: hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/master/RegionManager.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/master/RegionManager.java?rev=721411&r1=721410&r2=721411&view=diff
==============================================================================
--- hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/master/RegionManager.java
(original)
+++ hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/master/RegionManager.java
Fri Nov 28 02:14:21 2008
@@ -996,6 +996,10 @@
     while (i.hasNext()) {
       Pair<HRegionInfo,HServerAddress> pair = i.next();
       if (addr.equals(pair.getSecond())) {
+        if (LOG.isDebugEnabled()) {
+          LOG.debug("sending MSG_REGION_COMPACT " + pair.getFirst() + " to " +
+              addr);
+        }
         returnMsgs.add(new HMsg(HMsg.Type.MSG_REGION_COMPACT, pair.getFirst()));
         i.remove();
       }
@@ -1004,6 +1008,10 @@
     while (i.hasNext()) {
       Pair<HRegionInfo,HServerAddress> pair = i.next();
       if (addr.equals(pair.getSecond())) {
+        if (LOG.isDebugEnabled()) {
+          LOG.debug("sending MSG_REGION_SPLIT " + pair.getFirst() + " to " +
+              addr);
+        }
         returnMsgs.add(new HMsg(HMsg.Type.MSG_REGION_SPLIT, pair.getFirst()));
         i.remove();
       }

Modified: hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java?rev=721411&r1=721410&r2=721411&view=diff
==============================================================================
--- hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java
(original)
+++ hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java
Fri Nov 28 02:14:21 2008
@@ -534,8 +534,7 @@
       byte [] startKey = this.regionInfo.getStartKey();
       byte [] endKey = this.regionInfo.getEndKey();
       if (HStoreKey.equalsTwoRowKeys(this.regionInfo,startKey, midKey)) {
-        LOG.debug("Startkey (" + startKey + ") and midkey + (" + 
-          midKey + ") are same, not splitting");
+        LOG.debug("Startkey and midkey are same, not splitting");
         return null;
       }
       if (HStoreKey.equalsTwoRowKeys(this.regionInfo,midKey, endKey)) {

Modified: hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java?rev=721411&r1=721410&r2=721411&view=diff
==============================================================================
--- hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
(original)
+++ hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
Fri Nov 28 02:14:21 2008
@@ -21,6 +21,8 @@
 
 import java.io.IOException;
 import java.lang.Thread.UncaughtExceptionHandler;
+import java.lang.management.ManagementFactory;
+import java.lang.management.MemoryUsage;
 import java.lang.reflect.Constructor;
 import java.net.InetSocketAddress;
 import java.util.ArrayList;
@@ -194,7 +196,7 @@
   
   // flag set after we're done setting up server threads (used for testing)
   protected volatile boolean isOnline;
-    
+
   /**
    * Starts a HRegionServer at the default location
    * @param conf
@@ -322,9 +324,29 @@
           }
           try {
             doMetrics();
-            this.serverInfo.setLoad(new HServerLoad(requestCount.get(),
-                onlineRegions.size(), this.metrics.storefiles.get(),
-                this.metrics.memcacheSizeMB.get()));
+            MemoryUsage memory =
+                ManagementFactory.getMemoryMXBean().getHeapMemoryUsage();
+            HServerLoad hsl = new HServerLoad(requestCount.get(), 
+              (int)(memory.getUsed()/1024/1024),
+              (int)(memory.getMax()/1024/1024));
+            for (HRegion r: onlineRegions.values()) {
+              byte[] name = r.getRegionName();
+              int stores = 0;
+              int storefiles = 0;
+              int memcacheSizeMB = (int)(r.memcacheSize.get()/1024/1024);
+              int storefileIndexSizeMB = 0;
+              synchronized (r.stores) {
+                stores += r.stores.size();
+                for (HStore store: r.stores.values()) {
+                  storefiles += store.getStorefilesCount();
+                  storefileIndexSizeMB += 
+                    (int)(store.getStorefilesIndexSize()/1024/1024);
+                }
+              }
+              hsl.addRegionInfo(name, stores, storefiles, memcacheSizeMB,
+                storefileIndexSizeMB);
+            }
+            this.serverInfo.setLoad(hsl);
             this.requestCount.set(0);
             HMsg msgs[] = hbaseMaster.regionServerReport(
               serverInfo, outboundArray, getMostLoadedRegions());
@@ -434,10 +456,8 @@
         housekeeping();
         sleeper.sleep(lastMsg);
       } // for
-    } catch (OutOfMemoryError error) {
-      abort();
-      LOG.fatal("Ran out of memory", error);
     } catch (Throwable t) {
+      checkOOME(t);
       LOG.fatal("Unhandled exception. Aborting...", t);
       abort();
     }
@@ -550,6 +570,7 @@
       isOnline = true;
     } catch (IOException e) {
       this.stopRequested.set(true);
+      checkOOME(e);
       isOnline = false;
       e = RemoteExceptionHandler.checkIOException(e); 
       LOG.fatal("Failed init", e);
@@ -558,6 +579,22 @@
       throw ex;
     }
   }
+  
+  /*
+   * Check if an OOME and if so, call abort.
+   * @param e
+   * @return True if we OOME'd and are aborting.
+   */
+  private boolean checkOOME(final Throwable e) {
+    boolean aborting = false;
+    if (e instanceof OutOfMemoryError ||
+        (e.getCause()!= null && e.getCause() instanceof OutOfMemoryError)) {
+      LOG.fatal("OOME, aborting.", e);
+      abort();
+      aborting = true;
+    }
+    return aborting;
+  }
 
   /*
    * Thread to shutdown the region server in an orderly manner.  This thread
@@ -591,7 +628,7 @@
    */
   private static class MajorCompactionChecker extends Chore {
     private final HRegionServer instance;
-    
+
     MajorCompactionChecker(final HRegionServer h,
         final int sleepTime, final AtomicBoolean stopper) {
       super(sleepTime, stopper);
@@ -800,8 +837,9 @@
    * from under hbase or we OOME.
    */
   public void abort() {
-    reservedSpace.clear();
     this.abortRequested = true;
+    this.reservedSpace.clear();
+    LOG.info("Dump of metrics: " + this.metrics.toString());
     stop();
   }
 
@@ -846,7 +884,13 @@
     while(!stopRequested.get()) {
       try {
         this.requestCount.set(0);
-        this.serverInfo.setLoad(new HServerLoad(0, onlineRegions.size(), 0, 0));
+        MemoryUsage memory =
+          ManagementFactory.getMemoryMXBean().getHeapMemoryUsage();
+        HServerLoad hsl = new HServerLoad(0, (int)memory.getUsed()/1024/1024,
+          (int)memory.getMax()/1024/1024);
+        this.serverInfo.setLoad(hsl);
+        if (LOG.isDebugEnabled())
+          LOG.debug("sending initial server load: " + hsl);
         lastMsg = System.currentTimeMillis();
         result = this.hbaseMaster.regionServerStartup(serverInfo);
         break;
@@ -892,7 +936,6 @@
    */
   void reportSplit(HRegionInfo oldRegion, HRegionInfo newRegionA,
       HRegionInfo newRegionB) {
-
     outboundMsgs.add(new HMsg(HMsg.Type.MSG_REPORT_SPLIT, oldRegion,
       (oldRegion.getRegionNameAsString() + " split; daughters: " +
         newRegionA.getRegionNameAsString() + ", " +
@@ -975,7 +1018,11 @@
             case MSG_REGION_SPLIT: {
               // Force split a region
               HRegion region = getRegion(info.getRegionName());
+              // flush the memcache for the region
+              region.flushcache();
+              // flag that the region should be split
               region.regionInfo.shouldSplit(true);
+              // force a compaction
               compactSplitThread.compactionRequested(region,
                 "MSG_REGION_SPLIT");
             } break;
@@ -983,6 +1030,9 @@
             case MSG_REGION_COMPACT: {
               // Compact a region
               HRegion region = getRegion(info.getRegionName());
+              // flush the memcache for the region
+              region.flushcache();
+              // force a compaction
               compactSplitThread.compactionRequested(region,
                 "MSG_REGION_COMPACT");
             } break;
@@ -1017,6 +1067,7 @@
           }
         }
       } catch(Throwable t) {
+        checkOOME(t);
         LOG.fatal("Unhandled exception", t);
       } finally {
         LOG.info("worker thread exiting");
@@ -1039,8 +1090,9 @@
         this.compactSplitThread.
           compactionRequested(region, "Region open check");
       } catch (IOException e) {
-        LOG.error("error opening region " + regionInfo.getRegionNameAsString(), e);
-
+        checkOOME(e);
+        LOG.error("error opening region " + regionInfo.getRegionNameAsString(),
+          e);
         // TODO: add an extra field in HRegionInfo to indicate that there is
         // an error. We can't do that now because that would be an incompatible
         // change that would require a migration
@@ -1113,6 +1165,7 @@
         LOG.error("error closing region " +
             Bytes.toString(region.getRegionName()),
           RemoteExceptionHandler.checkIOException(e));
+        checkOOME(e);
       }
     }
     return regionsToClose;
@@ -1228,11 +1281,14 @@
       HRegion region = getRegion(regionName);
       Map<byte [], Cell> map = region.getFull(row, columnSet, ts,
           getLockFromId(lockId));
+      if (map == null || map.isEmpty())
+        return null;
       HbaseMapWritable<byte [], Cell> result =
         new HbaseMapWritable<byte [], Cell>();
       result.putAll(map);
       return new RowResult(row, result);
     } catch (IOException e) {
+      checkOOME(e);
       checkFileSystem();
       throw e;
     }
@@ -1250,6 +1306,7 @@
       RowResult rr = region.getClosestRowBefore(row, columnFamily);
       return rr;
     } catch (IOException e) {
+      checkOOME(e);
       checkFileSystem();
       throw e;
     }
@@ -1286,6 +1343,7 @@
       }
       return resultSets.toArray(new RowResult[resultSets.size()]);
     } catch (IOException e) {
+      checkOOME(e);
       checkFileSystem();
       throw e;
     }
@@ -1304,10 +1362,8 @@
     try {
       cacheFlusher.reclaimMemcacheMemory();
       region.batchUpdate(b, getLockFromId(b.getRowLock()));
-    } catch (OutOfMemoryError error) {
-      abort();
-      LOG.fatal("Ran out of memory", error);
     } catch (IOException e) {
+      checkOOME(e);
       checkFileSystem();
       throw e;
     }
@@ -1327,14 +1383,12 @@
         locks[i] = getLockFromId(b[i].getRowLock());
         region.batchUpdate(b[i], locks[i]);
       }
-    } catch (OutOfMemoryError error) {
-      abort();
-      LOG.fatal("Ran out of memory", error);
     } catch(WrongRegionException ex) {
       return i;
     } catch (NotServingRegionException ex) {
       return i;
     } catch (IOException e) {
+      checkOOME(e);
       checkFileSystem();
       throw e;
     }
@@ -1397,7 +1451,8 @@
       return scannerId;
     } catch (IOException e) {
       LOG.error("Error opening scanner (fsOk: " + this.fsOk + ")",
-          RemoteExceptionHandler.checkIOException(e));
+        RemoteExceptionHandler.checkIOException(e));
+      checkOOME(e);
       checkFileSystem();
       throw e;
     }
@@ -1430,6 +1485,9 @@
       s.close();
       this.leases.cancelLease(scannerName);
     } catch (IOException e) {
+      // TODO: Should we even be returning an exception out of a close?
+      // What can the client do with an exception in close?
+      checkOOME(e);
       checkFileSystem();
       throw e;
     }
@@ -1527,7 +1585,8 @@
       return lockId;
     } catch (IOException e) {
       LOG.error("Error obtaining row lock (fsOk: " + this.fsOk + ")",
-          RemoteExceptionHandler.checkIOException(e));
+        RemoteExceptionHandler.checkIOException(e));
+      checkOOME(e);
       checkFileSystem();
       throw e;
     }
@@ -1842,7 +1901,7 @@
   }
 
   public long getProtocolVersion(final String protocol, 
-      @SuppressWarnings("unused") final long clientVersion)
+      final long clientVersion)
   throws IOException {  
     if (protocol.equals(HRegionInterface.class.getName())) {
       return HBaseRPCProtocolVersion.versionID;
@@ -1974,4 +2033,4 @@
         .getClass(HConstants.REGION_SERVER_IMPL, HRegionServer.class);
     doMain(args, regionServerClass);
   }
-}
\ No newline at end of file
+}

Modified: hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/regionserver/HStore.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/regionserver/HStore.java?rev=721411&r1=721410&r2=721411&view=diff
==============================================================================
--- hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/regionserver/HStore.java
(original)
+++ hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/regionserver/HStore.java
Fri Nov 28 02:14:21 2008
@@ -823,6 +823,7 @@
       List<HStoreFile> filesToCompact = null;
       synchronized (storefiles) {
         if (this.storefiles.size() <= 0) {
+          LOG.debug("no store files to compact");
           return null;
         }
         // filesToCompact are sorted oldest to newest.
@@ -839,8 +840,8 @@
         doMajorCompaction = isMajorCompaction(filesToCompact);
       }
       boolean references = hasReferences(filesToCompact);
-      if (!doMajorCompaction && !references &&
-          filesToCompact.size() < compactionThreshold) {
+      if (!doMajorCompaction && !references && 
+          (forceSplit || (filesToCompact.size() < compactionThreshold))) {
         return checkSplit(forceSplit);
       }
       if (!fs.exists(compactionDir) && !fs.mkdirs(compactionDir)) {
@@ -2036,6 +2037,9 @@
             splitable = !curHSF.isReference();
             if (!splitable) {
               // RETURN IN MIDDLE OF FUNCTION!!! If not splitable, just return.
+              if (LOG.isDebugEnabled()) {
+                LOG.debug(curHSF +  " is not splittable");
+              }
               return null;
             }
           }
@@ -2063,6 +2067,9 @@
         // (ever) split this region. 
         if (HStoreKey.equalsTwoRowKeys(info, mk.getRow(), firstKey.getRow()) && 
             HStoreKey.equalsTwoRowKeys(info, mk.getRow(), lastKey.getRow())) {
+          if (LOG.isDebugEnabled()) {
+            LOG.debug("cannot split because midkey is the same as first or last row");
+          }
           return null;
         }
         return new StoreSize(maxSize, mk.getRow());



Mime
View raw message