hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jdcry...@apache.org
Subject svn commit: r931241 - in /hadoop/hbase/branches/0.20_pre_durability: ./ lib/ src/java/org/apache/hadoop/hbase/ src/java/org/apache/hadoop/hbase/client/ src/java/org/apache/hadoop/hbase/mapred/ src/java/org/apache/hadoop/hbase/mapreduce/ src/java/org/ap...
Date Tue, 06 Apr 2010 17:46:30 GMT
Author: jdcryans
Date: Tue Apr  6 17:46:29 2010
New Revision: 931241

URL: http://svn.apache.org/viewvc?rev=931241&view=rev
Log:
Adding the following jiras that were added after rev 919707 and that are needed:
   HBASE-2034  Client sync block can cause 1 thread of a multi-threaded client
               to block all others (Karthik Ranganathan via Stack)
   HBASE-2355  Unsynchronized logWriters map is mutated from several threads in
               HLog splitting (Todd Lipcon via Andrew Purtell)
   HBASE-2358  Store doReconstructionLog will fail if oldlogfile.log is
               empty and won't load region (Cosmin Lehene via Stack)
   HBASE-2365  Double-assignment around split
   HBASE-2174  Stop from resolving HRegionServer addresses to names using DNS on
               every heartbeat (Karthik Ranganathan via Stack)
   HBASE-2087  The wait on compaction because "Too many store files" 
               holds up all flushing
   HBASE-2252  Mapping a very big table kills region servers
   HBASE-2277  Update branch to hadoop 0.20.2

Added:
    hadoop/hbase/branches/0.20_pre_durability/lib/hadoop-0.20.2-core.jar   (with props)
    hadoop/hbase/branches/0.20_pre_durability/lib/hadoop-0.20.2-test.jar   (with props)
Removed:
    hadoop/hbase/branches/0.20_pre_durability/lib/hadoop-0.20.1-hdfs127-core.jar
    hadoop/hbase/branches/0.20_pre_durability/lib/hadoop-0.20.1-test.jar
Modified:
    hadoop/hbase/branches/0.20_pre_durability/CHANGES.txt
    hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/HServerInfo.java
    hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/client/HConnectionManager.java
    hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/mapred/TableInputFormatBase.java
    hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/mapreduce/TableInputFormat.java
    hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/master/ProcessRegionOpen.java
    hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/master/ProcessServerShutdown.java
    hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/master/RegionManager.java
    hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/master/ServerManager.java
    hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/regionserver/HLog.java
    hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java
    hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/regionserver/Store.java

Modified: hadoop/hbase/branches/0.20_pre_durability/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_pre_durability/CHANGES.txt?rev=931241&r1=931240&r2=931241&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20_pre_durability/CHANGES.txt (original)
+++ hadoop/hbase/branches/0.20_pre_durability/CHANGES.txt Tue Apr  6 17:46:29 2010
@@ -28,6 +28,13 @@ Release 0.20.4 - Unreleased
                half-written hfile (Ruslan Salyakhov via Stack)
    HBASE-2288  Shell fails on alter
    HBASE-2293  CME in RegionManager#isMetaServer
+   HBASE-2034  Client sync block can cause 1 thread of a multi-threaded client
+               to block all others (Karthik Ranganathan via Stack)
+   HBASE-2355  Unsynchronized logWriters map is mutated from several threads in
+               HLog splitting (Todd Lipcon via Andrew Purtell)
+   HBASE-2358  Store doReconstructionLog will fail if oldlogfile.log is
+               empty and won't load region (Cosmin Lehene via Stack)
+   HBASE-2365  Double-assignment around split
 
   IMPROVEMENTS
    HBASE-2180  Bad read performance from synchronizing hfile.fddatainputstream
@@ -42,6 +49,12 @@ Release 0.20.4 - Unreleased
                of the supplied byte array (Bruno Dumon via Stack)
    HBASE-2241  Change balancer sloppyness from 0.1 to 0.3
    HBASE-2262  ZKW.ensureExists should check for existence
+   HBASE-2174  Stop from resolving HRegionServer addresses to names using DNS on
+               every heartbeat (Karthik Ranganathan via Stack)
+   HBASE-2087  The wait on compaction because "Too many store files" 
+               holds up all flushing
+   HBASE-2252  Mapping a very big table kills region servers
+   HBASE-2277  Update branch to hadoop 0.20.2
 
   NEW FEATURES
    HBASE-2257  [stargate] multiuser mode

Added: hadoop/hbase/branches/0.20_pre_durability/lib/hadoop-0.20.2-core.jar
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_pre_durability/lib/hadoop-0.20.2-core.jar?rev=931241&view=auto
==============================================================================
Binary file - no diff available.

Propchange: hadoop/hbase/branches/0.20_pre_durability/lib/hadoop-0.20.2-core.jar
------------------------------------------------------------------------------
    svn:mime-type = application/octet-stream

Added: hadoop/hbase/branches/0.20_pre_durability/lib/hadoop-0.20.2-test.jar
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_pre_durability/lib/hadoop-0.20.2-test.jar?rev=931241&view=auto
==============================================================================
Binary file - no diff available.

Propchange: hadoop/hbase/branches/0.20_pre_durability/lib/hadoop-0.20.2-test.jar
------------------------------------------------------------------------------
    svn:mime-type = application/octet-stream

Modified: hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/HServerInfo.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/HServerInfo.java?rev=931241&r1=931240&r2=931241&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/HServerInfo.java
(original)
+++ hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/HServerInfo.java
Tue Apr  6 17:46:29 2010
@@ -137,8 +137,15 @@ public class HServerInfo implements Writ
    */
   public synchronized String getServerName() {
     if (this.serverName == null) {
+      // if we have the hostname of the RS, use it
+      if(this.name != null) {
+        this.serverName = getServerName(this.name, this.serverAddress.getPort(), this.startCode);
+      }
+      // go to DNS name resolution only if we dont have the name of the RS
+      else {
       this.serverName = getServerName(this.serverAddress, this.startCode);
     }
+    }
     return this.serverName;
   }
   
@@ -219,7 +226,7 @@ public class HServerInfo implements Writ
    * @param info
    * @return the server name in the form hostname_startcode_port
    */
-  public static String getServerName(HServerInfo info) {
+  private static String getServerName(HServerInfo info) {
     return getServerName(info.getServerAddress(), info.getStartCode());
   }
   

Modified: hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/client/HConnectionManager.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/client/HConnectionManager.java?rev=931241&r1=931240&r2=931241&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/client/HConnectionManager.java
(original)
+++ hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/client/HConnectionManager.java
Tue Apr  6 17:46:29 2010
@@ -627,16 +627,12 @@ public class HConnectionManager implemen
           return this.rootRegionLocation;
         }        
       } else if (Bytes.equals(tableName, META_TABLE_NAME)) {
-        synchronized (metaRegionLock) {
-          // This block guards against two threads trying to load the meta 
-          // region at the same time. The first will load the meta region and
-          // the second will use the value that the first one found.
-          return locateRegionInMeta(ROOT_TABLE_NAME, tableName, row, useCache);
-        }
+        return locateRegionInMeta(ROOT_TABLE_NAME, tableName, row, useCache,
+                                  metaRegionLock);
       } else {
-        synchronized(userRegionLock){
-          return locateRegionInMeta(META_TABLE_NAME, tableName, row, useCache);
-        }
+        // Region not in the cache - have to go to the meta RS
+        return locateRegionInMeta(META_TABLE_NAME, tableName, row, useCache,
+                                  userRegionLock);
       }
     }
 
@@ -645,18 +641,17 @@ public class HConnectionManager implemen
       * info that contains the table and row we're seeking.
       */
     private HRegionLocation locateRegionInMeta(final byte [] parentTable,
-      final byte [] tableName, final byte [] row, boolean useCache)
+      final byte [] tableName, final byte [] row, boolean useCache,
+      Object regionLockObject)
     throws IOException {
       HRegionLocation location = null;
-      // If supposed to be using the cache, then check it for a possible hit.
-      // Otherwise, delete any existing cached location so it won't interfere.
+      // If we are supposed to be using the cache, look in the cache to see if
+      // we already have the region.
       if (useCache) {
         location = getCachedLocation(tableName, row);
         if (location != null) {
           return location;
         }
-      } else {
-        deleteCachedLocation(tableName, row);
       }
       
       // build the key of the meta region we should be looking for.
@@ -676,10 +671,29 @@ public class HConnectionManager implemen
           HRegionInterface server =
             getHRegionConnection(metaLocation.getServerAddress());
 
+          Result regionInfoRow = null;
+          // This block guards against two threads trying to load the meta
+          // region at the same time. The first will load the meta region and
+          // the second will use the value that the first one found.
+          synchronized(regionLockObject) {
+            // Check the cache again for a hit in case some other thread made the
+            // same query while we were waiting on the lock. If not supposed to
+            // be using the cache, delete any existing cached location so it won't
+            // interfere.
+            if (useCache) {
+              location = getCachedLocation(tableName, row);
+              if (location != null) {
+                return location;
+              }
+            } else {
+              deleteCachedLocation(tableName, row);
+            }
+
           // Query the root or meta region for the location of the meta region
-          Result regionInfoRow = server.getClosestRowBefore(
+            regionInfoRow = server.getClosestRowBefore(
             metaLocation.getRegionInfo().getRegionName(), metaKey,
             HConstants.CATALOG_FAMILY);
+          }
           if (regionInfoRow == null) {
             throw new TableNotFoundException(Bytes.toString(tableName));
           }

Modified: hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/mapred/TableInputFormatBase.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/mapred/TableInputFormatBase.java?rev=931241&r1=931240&r2=931241&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/mapred/TableInputFormatBase.java
(original)
+++ hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/mapred/TableInputFormatBase.java
Tue Apr  6 17:46:29 2010
@@ -110,6 +110,7 @@ implements InputFormat<ImmutableBytesWri
           Scan scan = new Scan(firstRow, endRow);
           scan.addColumns(trrInputColumns);
           scan.setOldFilter(trrRowFilter);
+          scan.setCacheBlocks(false);
           this.scanner = this.htable.getScanner(scan);
         } else {
           LOG.debug("TIFB.restart, firstRow: " +

Modified: hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/mapreduce/TableInputFormat.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/mapreduce/TableInputFormat.java?rev=931241&r1=931240&r2=931241&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/mapreduce/TableInputFormat.java
(original)
+++ hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/mapreduce/TableInputFormat.java
Tue Apr  6 17:46:29 2010
@@ -119,13 +119,12 @@ implements Configurable {
           scan.setMaxVersions(Integer.parseInt(conf.get(SCAN_MAXVERSIONS)));
         }
         
-        if (conf.get(SCAN_CACHEBLOCKS) != null) {
-          scan.setCacheBlocks(Boolean.parseBoolean(conf.get(SCAN_CACHEBLOCKS)));
-        }
-
         if (conf.get(SCAN_CACHEDROWS) != null) {
           scan.setCaching(Integer.parseInt(conf.get(SCAN_CACHEDROWS)));
         }
+
+        // false by default, full table scans generate too much BC churn
+        scan.setCacheBlocks((conf.getBoolean(SCAN_CACHEBLOCKS, false)));
       } catch (Exception e) {
           LOG.error(StringUtils.stringifyException(e));
       }

Modified: hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/master/ProcessRegionOpen.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/master/ProcessRegionOpen.java?rev=931241&r1=931240&r2=931241&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/master/ProcessRegionOpen.java
(original)
+++ hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/master/ProcessRegionOpen.java
Tue Apr  6 17:46:29 2010
@@ -53,7 +53,7 @@ class ProcessRegionOpen extends ProcessR
 
   @Override
   public String toString() {
-    return "PendingOpenOperation from " + HServerInfo.getServerName(serverInfo);
+    return "PendingOpenOperation from " + serverInfo.getServerName();
   }
 
   @Override

Modified: hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/master/ProcessServerShutdown.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/master/ProcessServerShutdown.java?rev=931241&r1=931240&r2=931241&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/master/ProcessServerShutdown.java
(original)
+++ hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/master/ProcessServerShutdown.java
Tue Apr  6 17:46:29 2010
@@ -74,7 +74,7 @@ class ProcessServerShutdown extends Regi
    */
   public ProcessServerShutdown(HMaster master, HServerInfo serverInfo) {
     super(master);
-    this.deadServer = HServerInfo.getServerName(serverInfo);
+    this.deadServer = serverInfo.getServerName();
     this.deadServerAddress = serverInfo.getServerAddress();
     this.logSplit = false;
     this.rootRescanned = false;

Modified: hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/master/RegionManager.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/master/RegionManager.java?rev=931241&r1=931240&r2=931241&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/master/RegionManager.java
(original)
+++ hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/master/RegionManager.java
Tue Apr  6 17:46:29 2010
@@ -980,7 +980,7 @@ class RegionManager implements HConstant
         regionsInTransition.put(info.getRegionNameAsString(), s);
       }
     }
-    if (force || (!s.isPendingOpen() || !s.isOpen())) {
+    if (force || (!s.isPendingOpen() && !s.isOpen())) {
       s.setUnassigned();
     }
   }

Modified: hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/master/ServerManager.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/master/ServerManager.java?rev=931241&r1=931240&r2=931241&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/master/ServerManager.java
(original)
+++ hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/master/ServerManager.java
Tue Apr  6 17:46:29 2010
@@ -155,7 +155,7 @@ class ServerManager implements HConstant
   public void regionServerStartup(final HServerInfo serverInfo) 
   throws Leases.LeaseStillHeldException {
     HServerInfo info = new HServerInfo(serverInfo);
-    String serverName = HServerInfo.getServerName(info);
+    String serverName = info.getServerName();
     if (serversToServerInfo.containsKey(serverName) ||
         deadServers.contains(serverName)) {
       LOG.debug("Server start was rejected: " + serverInfo);
@@ -212,7 +212,7 @@ class ServerManager implements HConstant
    */
   public void recordNewServer(HServerInfo info, boolean useInfoLoad) {
     HServerLoad load = useInfoLoad ? info.getLoad() : new HServerLoad();
-    String serverName = HServerInfo.getServerName(info);
+    String serverName = info.getServerName();
     info.setLoad(load);
     // We must set this watcher here because it can be set on a fresh start
     // or on a failover
@@ -834,7 +834,7 @@ class ServerManager implements HConstant
         serverAddressToServerInfo.remove(serverAddress);
         HServerInfo info = serversToServerInfo.remove(server);
         if (info != null) {
-          String serverName = HServerInfo.getServerName(info);
+          String serverName = info.getServerName();
           HServerLoad load = serversToLoad.remove(serverName);
           if (load != null) {
             synchronized (loadToServers) {

Modified: hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/regionserver/HLog.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/regionserver/HLog.java?rev=931241&r1=931240&r2=931241&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/regionserver/HLog.java
(original)
+++ hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/regionserver/HLog.java
Tue Apr  6 17:46:29 2010
@@ -923,7 +923,8 @@ public class HLog implements HConstants,
     final HBaseConfiguration conf)
   throws IOException {
     final Map<byte [], WriterAndPath> logWriters =
-      new TreeMap<byte [], WriterAndPath>(Bytes.BYTES_COMPARATOR);
+      Collections.synchronizedMap(
+        new TreeMap<byte [], WriterAndPath>(Bytes.BYTES_COMPARATOR));
     List<Path> splits = null;
     
     // Number of threads to use when log splitting to rewrite the logs.
@@ -1177,7 +1178,7 @@ public class HLog implements HConstants,
    * @return the HLog directory name
    */
   public static String getHLogDirectoryName(HServerInfo info) {
-    return getHLogDirectoryName(HServerInfo.getServerName(info));
+    return getHLogDirectoryName(info.getServerName());
   }
 
   /*

Modified: hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java?rev=931241&r1=931240&r2=931241&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java
(original)
+++ hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java
Tue Apr  6 17:46:29 2010
@@ -222,7 +222,25 @@ class MemStoreFlusher extends Thread imp
    * not flushed.
    */
   private boolean flushRegion(HRegion region, boolean removeFromQueue) {
-    checkStoreFileCount(region);
+    // if removeFromQueue, then we come from flushSomeRegions and we need
+    // to block if there's too many store files. Else, we don't want to hang
+    // the main flushing thread so we'll just the region at the end of the
+    // queue if there's too many files.
+    if (removeFromQueue) {
+      checkStoreFileCount(region);
+    } else if (isTooManyStoreFiles(region)) {
+      LOG.warn("Region " + region.getRegionNameAsString() + " has too many " +
+          "store files, putting it back at the end of the flush queue.");
+      server.compactSplitThread.compactionRequested(region, getName());
+      // If there's only this item in the queue or they are all in this
+      // situation, we will loop at lot. Sleep a bit.
+      try {
+        Thread.sleep(1000);
+      } catch (InterruptedException e) { } // just continue
+      flushQueue.add(region);
+      // Tell a lie, it's not flushed but it's ok
+      return true;
+    }
     synchronized (regionsInQueue) {
       // See comment above for removeFromQueue on why we do not
       // take the region out of the set. If removeFromQueue is true, remove it
@@ -310,6 +328,15 @@ class MemStoreFlusher extends Thread imp
     }
   }
 
+  private boolean isTooManyStoreFiles(HRegion region) {
+    for (Store hstore: region.stores.values()) {
+      if (hstore.getStorefilesCount() > this.blockingStoreFilesNumber) {
+        return true;
+      }
+    }
+    return false;
+  }
+
   /**
    * Check if the regionserver's memstore memory usage is greater than the 
    * limit. If so, flush regions with the biggest memstores until we're down

Modified: hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/regionserver/Store.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/regionserver/Store.java?rev=931241&r1=931240&r2=931241&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/regionserver/Store.java
(original)
+++ hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/regionserver/Store.java
Tue Apr  6 17:46:29 2010
@@ -302,11 +302,11 @@ public class Store implements HConstants
       // Nothing to do.
       return -1;
     }
-    // Check its not empty.
-    FileStatus [] stats = this.fs.listStatus(reconstructionLog);
-    if (stats == null || stats.length == 0) {
+    FileStatus stat = this.fs.getFileStatus(reconstructionLog);
+    if (stat.getLen() <= 0) {
       LOG.warn("Passed reconstruction log " + reconstructionLog +
-        " is zero-length");
+        " is zero-length. Deleting existing file");
+      fs.delete(reconstructionLog, false);
       return -1;
     }
     // TODO: This could grow large and blow heap out.  Need to get it into



Mime
View raw message