hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jxi...@apache.org
Subject svn commit: r1417079 - in /hbase/branches/0.94/src: main/java/org/apache/hadoop/hbase/ main/java/org/apache/hadoop/hbase/regionserver/ test/java/org/apache/hadoop/hbase/regionserver/
Date Tue, 04 Dec 2012 18:16:23 GMT
Author: jxiang
Date: Tue Dec  4 18:16:21 2012
New Revision: 1417079

URL: http://svn.apache.org/viewvc?rev=1417079&view=rev
Log:
HBASE-6423 Writes should not block reads on blocking updates to memstores

Added:
    hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/RegionTooBusyException.java
  (with props)
    hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionBusyWait.java
  (with props)
Modified:
    hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
    hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java

Added: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/RegionTooBusyException.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/RegionTooBusyException.java?rev=1417079&view=auto
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/RegionTooBusyException.java
(added)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/RegionTooBusyException.java
Tue Dec  4 18:16:21 2012
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase;
+
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * Thrown by a region server if it will block and wait to serve a request.
+ * For example, the client wants to insert something to a region while the
+ * region is compacting.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public class RegionTooBusyException extends IOException {
+  private static final long serialVersionUID = 1728345723728342L;
+
+  /** default constructor */
+  public RegionTooBusyException() {
+    super();
+  }
+
+  /**
+   * Constructor
+   * @param msg message
+   */
+  public RegionTooBusyException(final String msg) {
+    super(msg);
+  }
+}

Propchange: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/RegionTooBusyException.java
------------------------------------------------------------------------------
    svn:eol-style = native

Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java?rev=1417079&r1=1417078&r2=1417079&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java Tue
Dec  4 18:16:21 2012
@@ -55,6 +55,7 @@ import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.locks.Lock;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
 
 import org.apache.commons.logging.Log;
@@ -76,6 +77,7 @@ import org.apache.hadoop.hbase.HRegionIn
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.NotServingRegionException;
+import org.apache.hadoop.hbase.RegionTooBusyException;
 import org.apache.hadoop.hbase.UnknownScannerException;
 import org.apache.hadoop.hbase.backup.HFileArchiver;
 import org.apache.hadoop.hbase.client.Append;
@@ -235,6 +237,25 @@ public class HRegion implements HeapSize
   final Configuration conf;
   final int rowLockWaitDuration;
   static final int DEFAULT_ROWLOCK_WAIT_DURATION = 30000;
+
+  // The internal wait duration to acquire a lock before read/update
+  // from the region. It is not per row. The purpose of this wait time
+  // is to avoid waiting a long time while the region is busy, so that
+  // we can release the IPC handler soon enough to improve the
+  // availability of the region server. It can be adjusted by
+  // tuning configuration "hbase.busy.wait.duration".
+  final long busyWaitDuration;
+  static final long DEFAULT_BUSY_WAIT_DURATION = HConstants.DEFAULT_HBASE_RPC_TIMEOUT;
+
+  // If updating multiple rows in one call, wait longer,
+  // i.e. waiting for busyWaitDuration * # of rows. However,
+  // we can limit the max multiplier.
+  final int maxBusyWaitMultiplier;
+
+  // Max busy wait duration. There is no point to wait longer than the RPC
+  // purge timeout, when a RPC call will be terminated by the RPC engine.
+  final long maxBusyWaitDuration;
+
   final HRegionInfo regionInfo;
   final Path regiondir;
   KeyValue.KVComparator comparator;
@@ -354,6 +375,10 @@ public class HRegion implements HeapSize
     this.coprocessorHost = null;
     this.scannerReadPoints = new ConcurrentHashMap<RegionScanner, Long>();
     this.opMetrics = new OperationMetrics();
+
+    this.maxBusyWaitDuration = 2 * HConstants.DEFAULT_HBASE_RPC_TIMEOUT;
+    this.busyWaitDuration = DEFAULT_BUSY_WAIT_DURATION;
+    this.maxBusyWaitMultiplier = 2;
   }
 
   /**
@@ -400,6 +425,17 @@ public class HRegion implements HeapSize
     this.scannerReadPoints = new ConcurrentHashMap<RegionScanner, Long>();
     this.opMetrics = new OperationMetrics(conf, this.regionInfo);
 
+    this.busyWaitDuration = conf.getLong(
+      "hbase.busy.wait.duration", DEFAULT_BUSY_WAIT_DURATION);
+    this.maxBusyWaitMultiplier = conf.getInt("hbase.busy.wait.multiplier.max", 2);
+    if (busyWaitDuration * maxBusyWaitMultiplier <= 0L) {
+      throw new IllegalArgumentException("Invalid hbase.busy.wait.duration ("
+        + busyWaitDuration + ") or hbase.busy.wait.multiplier.max ("
+        + maxBusyWaitMultiplier + "). Their product should be positive");
+    }
+    this.maxBusyWaitDuration = conf.getLong("ipc.client.call.purge.timeout",
+      2 * HConstants.DEFAULT_HBASE_RPC_TIMEOUT);
+
     /*
      * timestamp.slop provides a server-side constraint on the timestamp. This
      * assumes that you base your TS around currentTimeMillis(). In this case,
@@ -883,6 +919,7 @@ public class HRegion implements HeapSize
 
     this.closing.set(true);
     status.setStatus("Disabling writes for close");
+    // block waiting for the lock for closing
     lock.writeLock().lock();
     try {
       if (this.isClosed()) {
@@ -1192,6 +1229,7 @@ public class HRegion implements HeapSize
       return false;
     }
     Preconditions.checkArgument(cr.getHRegion().equals(this));
+    // block waiting for the lock for compaction
     lock.readLock().lock();
     MonitoredTask status = TaskMonitor.get().createStatus(
         "Compacting " + cr.getStore() + " in " + this);
@@ -1271,7 +1309,7 @@ public class HRegion implements HeapSize
     }
     MonitoredTask status = TaskMonitor.get().createStatus("Flushing " + this);
     status.setStatus("Acquiring readlock on region");
-    lock.readLock().lock();
+    lock(lock.readLock());
     try {
       if (this.closed.get()) {
         LOG.debug("Skipping flush on " + this + " because closed");
@@ -1406,6 +1444,7 @@ public class HRegion implements HeapSize
     // end up in both snapshot and memstore (makes it difficult to do atomic
     // rows then)
     status.setStatus("Obtaining lock to block concurrent updates");
+    // block waiting for the lock for internal flush
     this.updatesLock.writeLock().lock();
     long flushsize = this.memstoreSize.get();
     status.setStatus("Preparing to flush by snapshotting stores");
@@ -1784,7 +1823,7 @@ public class HRegion implements HeapSize
     byte [] byteNow = Bytes.toBytes(now);
     boolean flush = false;
 
-    updatesLock.readLock().lock();
+    lock(updatesLock.readLock());
     try {
       prepareDeleteTimestamps(delete.getFamilyMap(), byteNow);
 
@@ -2163,7 +2202,7 @@ public class HRegion implements HeapSize
         }
       }
 
-      this.updatesLock.readLock().lock();
+      lock(this.updatesLock.readLock(), numReadyToWrite);
       locked = true;
 
       //
@@ -2457,7 +2496,8 @@ public class HRegion implements HeapSize
    * this and the synchronize on 'this' inside in internalFlushCache to send
    * the notify.
    */
-  private void checkResources() {
+  private void checkResources()
+      throws RegionTooBusyException, InterruptedIOException {
 
     // If catalog region, do not impose resource constraints or block updates.
     if (this.getRegionInfo().isMetaRegion()) return;
@@ -2475,12 +2515,30 @@ public class HRegion implements HeapSize
           " is >= than blocking " +
           StringUtils.humanReadableInt(this.blockingMemStoreSize) + " size");
       }
+      long now = EnvironmentEdgeManager.currentTimeMillis();
+      long timeToWait = startTime + busyWaitDuration - now;
+      if (timeToWait <= 0L) {
+        final long totalTime = now - startTime;
+        this.updatesBlockedMs.add(totalTime);
+        LOG.info("Failed to unblock updates for region " + this + " '"
+          + Thread.currentThread().getName() + "' in " + totalTime
+          + "ms. The region is still busy.");
+        throw new RegionTooBusyException("region is flushing");
+      }
       blocked = true;
       synchronized(this) {
         try {
-          wait(threadWakeFrequency);
-        } catch (InterruptedException e) {
-          Thread.currentThread().interrupt();
+          wait(Math.min(timeToWait, threadWakeFrequency));
+        } catch (InterruptedException ie) {
+          final long totalTime = EnvironmentEdgeManager.currentTimeMillis() - startTime;
+          if (totalTime > 0) {
+            this.updatesBlockedMs.add(totalTime);
+          }
+          LOG.info("Interrupted while waiting to unblock updates for region "
+            + this + " '" + Thread.currentThread().getName() + "'");
+          InterruptedIOException iie = new InterruptedIOException();
+          iie.initCause(ie);
+          throw iie;
         }
       }
     }
@@ -2547,7 +2605,7 @@ public class HRegion implements HeapSize
     byte[] byteNow = Bytes.toBytes(now);
     boolean flush = false;
 
-    this.updatesLock.readLock().lock();
+    lock(this.updatesLock.readLock());
     try {
       checkFamilies(familyMap.keySet());
       checkTimestamps(familyMap, now);
@@ -3172,6 +3230,7 @@ public class HRegion implements HeapSize
    * @param lockId  The lock ID to release.
    */
   public void releaseRowLock(final Integer lockId) {
+    if (lockId == null) return; // null lock id, do nothing
     HashedBytes rowKey = lockIds.remove(lockId);
     if (rowKey == null) {
       LOG.warn("Release unknown lockId: " + lockId);
@@ -4303,7 +4362,7 @@ public class HRegion implements HeapSize
       }
 
       // 3. acquire the region lock
-      this.updatesLock.readLock().lock();
+      lock(this.updatesLock.readLock(), acquiredLocks.size());
       locked = true;
 
       // 4. Get a mvcc write number
@@ -4456,7 +4515,7 @@ public class HRegion implements HeapSize
     this.writeRequestsCount.increment();
     try {
       Integer lid = getLock(lockid, row, true);
-      this.updatesLock.readLock().lock();
+      lock(this.updatesLock.readLock());
       try {
         long now = EnvironmentEdgeManager.currentTimeMillis();
         // Process each family
@@ -4607,7 +4666,7 @@ public class HRegion implements HeapSize
     this.writeRequestsCount.increment();
     try {
       Integer lid = getLock(lockid, row, true);
-      this.updatesLock.readLock().lock();
+      lock(this.updatesLock.readLock());
       try {
         long now = EnvironmentEdgeManager.currentTimeMillis();
         // Process each family
@@ -4725,7 +4784,7 @@ public class HRegion implements HeapSize
     this.writeRequestsCount.increment();
     try {
       Integer lid = obtainRowLock(row);
-      this.updatesLock.readLock().lock();
+      lock(this.updatesLock.readLock());
       try {
         Store store = stores.get(family);
 
@@ -5119,13 +5178,16 @@ public class HRegion implements HeapSize
    * #closeRegionOperation needs to be called in the try's finally block
    * Acquires a read lock and checks if the region is closing or closed.
    * @throws NotServingRegionException when the region is closing or closed
+   * @throws RegionTooBusyException if failed to get the lock in time
+   * @throws InterruptedIOException if interrupted while waiting for a lock
    */
-  private void startRegionOperation() throws NotServingRegionException {
+  private void startRegionOperation()
+      throws NotServingRegionException, RegionTooBusyException, InterruptedIOException {
     if (this.closing.get()) {
       throw new NotServingRegionException(regionInfo.getRegionNameAsString() +
           " is closing");
     }
-    lock.readLock().lock();
+    lock(lock.readLock());
     if (this.closed.get()) {
       lock.readLock().unlock();
       throw new NotServingRegionException(regionInfo.getRegionNameAsString() +
@@ -5147,15 +5209,17 @@ public class HRegion implements HeapSize
    * #closeBulkRegionOperation needs to be called in the try's finally block
    * Acquires a writelock and checks if the region is closing or closed.
    * @throws NotServingRegionException when the region is closing or closed
+   * @throws RegionTooBusyException if failed to get the lock in time
+   * @throws InterruptedIOException if interrupted while waiting for a lock
    */
   private void startBulkRegionOperation(boolean writeLockNeeded)
-  throws NotServingRegionException {
+      throws NotServingRegionException, RegionTooBusyException, InterruptedIOException {
     if (this.closing.get()) {
       throw new NotServingRegionException(regionInfo.getRegionNameAsString() +
           " is closing");
     }
-    if (writeLockNeeded) lock.writeLock().lock();
-    else lock.readLock().lock();
+    if (writeLockNeeded) lock(lock.writeLock());
+    else lock(lock.readLock());
     if (this.closed.get()) {
       if (writeLockNeeded) lock.writeLock().unlock();
       else lock.readLock().unlock();
@@ -5168,7 +5232,7 @@ public class HRegion implements HeapSize
    * Closes the lock. This needs to be called in the finally block corresponding
    * to the try block of #startRegionOperation
    */
-  private void closeBulkRegionOperation(){
+  private void closeBulkRegionOperation() {
     if (lock.writeLock().isHeldByCurrentThread()) lock.writeLock().unlock();
     else lock.readLock().unlock();
   }
@@ -5193,6 +5257,33 @@ public class HRegion implements HeapSize
     dataInMemoryWithoutWAL.addAndGet(putSize);
   }
 
+  private void lock(final Lock lock)
+      throws RegionTooBusyException, InterruptedIOException {
+    lock(lock, 1);
+  }
+
+  /**
+   * Try to acquire a lock.  Throw RegionTooBusyException
+   * if failed to get the lock in time. Throw InterruptedIOException
+   * if interrupted while waiting for the lock.
+   */
+  private void lock(final Lock lock, final int multiplier)
+      throws RegionTooBusyException, InterruptedIOException {
+    try {
+      final long waitTime = Math.min(maxBusyWaitDuration,
+        busyWaitDuration * Math.min(multiplier, maxBusyWaitMultiplier));
+      if (!lock.tryLock(waitTime, TimeUnit.MILLISECONDS)) {
+        throw new RegionTooBusyException(
+          "failed to get a lock in " + waitTime + "ms");
+      }
+    } catch (InterruptedException ie) {
+      LOG.info("Interrupted while waiting for a lock");
+      InterruptedIOException iie = new InterruptedIOException();
+      iie.initCause(ie);
+      throw iie;
+    }
+  }
+
   /**
    * Calls sync with the given transaction ID if the region's table is not
    * deferring it.
@@ -5232,7 +5323,6 @@ public class HRegion implements HeapSize
     }
   };
 
-
   /**
    * Facility for dumping and compacting catalog tables.
    * Only does catalog tables since these are only tables we for sure know

Modified: hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java?rev=1417079&r1=1417078&r2=1417079&view=diff
==============================================================================
--- hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
(original)
+++ hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
Tue Dec  4 18:16:21 2012
@@ -21,6 +21,7 @@ package org.apache.hadoop.hbase.regionse
 
 
 import java.io.IOException;
+import java.io.InterruptedIOException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashMap;
@@ -51,17 +52,15 @@ import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.MediumTests;
 import org.apache.hadoop.hbase.MiniHBaseCluster;
 import org.apache.hadoop.hbase.MultithreadedTestUtil;
-import org.apache.hadoop.hbase.MultithreadedTestUtil.TestThread;
 import org.apache.hadoop.hbase.MultithreadedTestUtil.RepeatingTestThread;
+import org.apache.hadoop.hbase.MultithreadedTestUtil.TestThread;
 import org.apache.hadoop.hbase.client.Append;
 import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.client.Increment;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
 import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.filter.BinaryComparator;
 import org.apache.hadoop.hbase.filter.ColumnCountGetFilter;
@@ -71,8 +70,6 @@ import org.apache.hadoop.hbase.filter.Fi
 import org.apache.hadoop.hbase.filter.NullComparator;
 import org.apache.hadoop.hbase.filter.PrefixFilter;
 import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
-import org.apache.hadoop.hbase.io.hfile.Compression;
-import org.apache.hadoop.hbase.master.HMaster;
 import org.apache.hadoop.hbase.monitoring.MonitoredRPCHandler;
 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
 import org.apache.hadoop.hbase.monitoring.TaskMonitor;
@@ -105,6 +102,7 @@ import com.google.common.collect.Lists;
  * HRegions or in the HBaseMaster, so only basic testing is possible.
  */
 @Category(MediumTests.class)
+@SuppressWarnings("deprecation")
 public class TestHRegion extends HBaseTestCase {
   // Do not spin up clusters in here.  If you need to spin up a cluster, do it
   // over in TestHRegionOnCluster.
@@ -157,7 +155,6 @@ public class TestHRegion extends HBaseTe
     String method = "testCompactionAffectedByScanners";
     byte[] tableName = Bytes.toBytes(method);
     byte[] family = Bytes.toBytes("family");
-    Configuration conf = HBaseConfiguration.create();
     this.region = initHRegion(tableName, method, conf, family);
 
     Put put = new Put(Bytes.toBytes("r1"));
@@ -209,7 +206,6 @@ public class TestHRegion extends HBaseTe
     String method = "testToShowNPEOnRegionScannerReseek";
     byte[] tableName = Bytes.toBytes(method);
     byte[] family = Bytes.toBytes("family");
-    Configuration conf = HBaseConfiguration.create();
     this.region = initHRegion(tableName, method, conf, family);
 
     Put put = new Put(Bytes.toBytes("r1"));
@@ -242,7 +238,6 @@ public class TestHRegion extends HBaseTe
     String method = "testSkipRecoveredEditsReplay";
     byte[] tableName = Bytes.toBytes(method);
     byte[] family = Bytes.toBytes("family");
-    Configuration conf = HBaseConfiguration.create();
     this.region = initHRegion(tableName, method, conf, family);
     try {
       Path regiondir = region.getRegionDir();
@@ -288,7 +283,7 @@ public class TestHRegion extends HBaseTe
     String method = "testSkipRecoveredEditsReplaySomeIgnored";
     byte[] tableName = Bytes.toBytes(method);
     byte[] family = Bytes.toBytes("family");
-    this.region = initHRegion(tableName, method, HBaseConfiguration.create(), family);
+    this.region = initHRegion(tableName, method, conf, family);
     try {
       Path regiondir = region.getRegionDir();
       FileSystem fs = region.getFilesystem();
@@ -338,7 +333,7 @@ public class TestHRegion extends HBaseTe
     String method = "testSkipRecoveredEditsReplayAllIgnored";
     byte[] tableName = Bytes.toBytes(method);
     byte[] family = Bytes.toBytes("family");
-    this.region = initHRegion(tableName, method, HBaseConfiguration.create(), family);
+    this.region = initHRegion(tableName, method, conf, family);
     try {
       Path regiondir = region.getRegionDir();
       FileSystem fs = region.getFilesystem();
@@ -464,7 +459,7 @@ public class TestHRegion extends HBaseTe
     byte[][] FAMILIES = new byte[][] { Bytes.toBytes("trans-blob"),
         Bytes.toBytes("trans-type"), Bytes.toBytes("trans-date"),
         Bytes.toBytes("trans-tags"), Bytes.toBytes("trans-group") };
-    this.region = initHRegion(TABLE, getName(), FAMILIES);
+    this.region = initHRegion(TABLE, getName(), conf, FAMILIES);
     try {
       String value = "this is the value";
       String value2 = "this is some other value";
@@ -585,7 +580,7 @@ public class TestHRegion extends HBaseTe
   public void testFamilyWithAndWithoutColon() throws Exception {
     byte [] b = Bytes.toBytes(getName());
     byte [] cf = Bytes.toBytes(COLUMN_FAMILY);
-    this.region = initHRegion(b, getName(), cf);
+    this.region = initHRegion(b, getName(), conf, cf);
     try {
       Put p = new Put(b);
       byte [] cfwithcolon = Bytes.toBytes(COLUMN_FAMILY + ":");
@@ -609,7 +604,7 @@ public class TestHRegion extends HBaseTe
     byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
     byte[] qual = Bytes.toBytes("qual");
     byte[] val = Bytes.toBytes("val");
-    this.region = initHRegion(b, getName(), cf);
+    this.region = initHRegion(b, getName(), conf, cf);
     try {
       HLog.getSyncTime(); // clear counter from prior tests
       assertEquals(0, HLog.getSyncTime().count);
@@ -643,7 +638,7 @@ public class TestHRegion extends HBaseTe
       Integer lockedRow = region.obtainRowLock(Bytes.toBytes("row_2"));
 
       MultithreadedTestUtil.TestContext ctx =
-        new MultithreadedTestUtil.TestContext(HBaseConfiguration.create());
+        new MultithreadedTestUtil.TestContext(conf);
       final AtomicReference<OperationStatus[]> retFromThread =
         new AtomicReference<OperationStatus[]>();
       TestThread putter = new TestThread(ctx) {
@@ -710,9 +705,7 @@ public class TestHRegion extends HBaseTe
     byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
     byte[] qual = Bytes.toBytes("qual");
     byte[] val = Bytes.toBytes("val");
-
-    HBaseConfiguration conf = new HBaseConfiguration();
-
+    Configuration conf = HBaseConfiguration.create(this.conf);
 
     // add data with a timestamp that is too recent for range. Ensure assert
     conf.setInt("hbase.hregion.keyvalue.timestamp.slop.millisecs", 1000);
@@ -759,7 +752,7 @@ public class TestHRegion extends HBaseTe
 
     //Setting up region
     String method = this.getName();
-    this.region = initHRegion(tableName, method, fam1);
+    this.region = initHRegion(tableName, method, conf, fam1);
     try {
       //Putting empty data in key
       Put put = new Put(row1);
@@ -834,7 +827,7 @@ public class TestHRegion extends HBaseTe
 
     //Setting up region
     String method = this.getName();
-    this.region = initHRegion(tableName, method, fam1);
+    this.region = initHRegion(tableName, method, conf, fam1);
     try {
       //Putting data in key
       Put put = new Put(row1);
@@ -868,7 +861,7 @@ public class TestHRegion extends HBaseTe
 
     //Setting up region
     String method = this.getName();
-    this.region = initHRegion(tableName, method, fam1);
+    this.region = initHRegion(tableName, method, conf, fam1);
     try {
       //Putting data in key
       Put put = new Put(row1);
@@ -906,7 +899,7 @@ public class TestHRegion extends HBaseTe
 
     //Setting up region
     String method = this.getName();
-    this.region = initHRegion(tableName, method, families);
+    this.region = initHRegion(tableName, method, conf, families);
     try {
       //Putting data in the key to check
       Put put = new Put(row1);
@@ -945,7 +938,7 @@ public class TestHRegion extends HBaseTe
   }
 
   public void testCheckAndPut_wrongRowInPut() throws IOException {
-    this.region = initHRegion(tableName, this.getName(), COLUMNS);
+    this.region = initHRegion(tableName, this.getName(), conf, COLUMNS);
     try {
       Put put = new Put(row2);
       put.add(fam1, qual1, value1);
@@ -980,7 +973,7 @@ public class TestHRegion extends HBaseTe
 
     //Setting up region
     String method = this.getName();
-    this.region = initHRegion(tableName, method, families);
+    this.region = initHRegion(tableName, method, conf, families);
     try {
       //Put content
       Put put = new Put(row1);
@@ -1055,7 +1048,7 @@ public class TestHRegion extends HBaseTe
     put.add(fam1, qual, 2, value);
 
     String method = this.getName();
-    this.region = initHRegion(tableName, method, fam1);
+    this.region = initHRegion(tableName, method, conf, fam1);
     try {
       region.put(put);
 
@@ -1085,7 +1078,7 @@ public class TestHRegion extends HBaseTe
 
     //Setting up region
     String method = this.getName();
-    this.region = initHRegion(tableName, method, fam1, fam2, fam3);
+    this.region = initHRegion(tableName, method, conf, fam1, fam2, fam3);
     try {
       List<KeyValue> kvs  = new ArrayList<KeyValue>();
       kvs.add(new KeyValue(row1, fam4, null, null));
@@ -1123,7 +1116,7 @@ public class TestHRegion extends HBaseTe
     byte [] fam = Bytes.toBytes("info");
     byte [][] families = {fam};
     String method = this.getName();
-    this.region = initHRegion(tableName, method, families);
+    this.region = initHRegion(tableName, method, conf, families);
     try {
       EnvironmentEdgeManagerTestHelper.injectEdge(new IncrementingEnvironmentEdge());
 
@@ -1191,7 +1184,7 @@ public class TestHRegion extends HBaseTe
     byte [] fam = Bytes.toBytes("info");
     byte [][] families = {fam};
     String method = this.getName();
-    this.region = initHRegion(tableName, method, families);
+    this.region = initHRegion(tableName, method, conf, families);
     try {
       byte [] row = Bytes.toBytes("table_name");
       // column names
@@ -1234,7 +1227,7 @@ public class TestHRegion extends HBaseTe
     byte [] fam = Bytes.toBytes("info");
     byte [][] families = {fam};
     String method = this.getName();
-    this.region = initHRegion(tableName, method, families);
+    this.region = initHRegion(tableName, method, conf, families);
     try {
       byte [] row = Bytes.toBytes("row1");
       // column names
@@ -1287,7 +1280,7 @@ public class TestHRegion extends HBaseTe
     byte[] fam = Bytes.toBytes("info");
     byte[][] families = { fam };
     String method = this.getName();
-    HBaseConfiguration conf = new HBaseConfiguration();
+    Configuration conf = HBaseConfiguration.create(this.conf);
 
     // add data with a timestamp that is too recent for range. Ensure assert
     conf.setInt("hbase.hregion.keyvalue.timestamp.slop.millisecs", 1000);
@@ -1318,7 +1311,7 @@ public class TestHRegion extends HBaseTe
     byte [] tableName = Bytes.toBytes("test_table");
     byte [] fam1 = Bytes.toBytes("columnA");
     byte [] fam2 = Bytes.toBytes("columnB");
-    this.region = initHRegion(tableName, getName(), fam1, fam2);
+    this.region = initHRegion(tableName, getName(), conf, fam1, fam2);
     try {
       byte [] rowA = Bytes.toBytes("rowA");
       byte [] rowB = Bytes.toBytes("rowB");
@@ -1371,7 +1364,7 @@ public class TestHRegion extends HBaseTe
 
   public void doTestDelete_AndPostInsert(Delete delete)
       throws IOException, InterruptedException {
-    this.region = initHRegion(tableName, getName(), fam1);
+    this.region = initHRegion(tableName, getName(), conf, fam1);
     try {
       EnvironmentEdgeManagerTestHelper.injectEdge(new IncrementingEnvironmentEdge());
       Put put = new Put(row);
@@ -1424,7 +1417,7 @@ public class TestHRegion extends HBaseTe
 
     //Setting up region
     String method = this.getName();
-    this.region = initHRegion(tableName, method, fam1);
+    this.region = initHRegion(tableName, method, conf, fam1);
     try {
       //Building checkerList
       List<KeyValue> kvs  = new ArrayList<KeyValue>();
@@ -1464,7 +1457,7 @@ public class TestHRegion extends HBaseTe
 
     //Setting up region
     String method = this.getName();
-    this.region = initHRegion(tableName, method, fam1);
+    this.region = initHRegion(tableName, method, conf, fam1);
     try {
       Get get = new Get(row1);
       get.addColumn(fam2, col1);
@@ -1495,7 +1488,7 @@ public class TestHRegion extends HBaseTe
 
     //Setting up region
     String method = this.getName();
-    this.region = initHRegion(tableName, method, fam1);
+    this.region = initHRegion(tableName, method, conf, fam1);
     try {
       //Add to memstore
       Put put = new Put(row1);
@@ -1545,7 +1538,7 @@ public class TestHRegion extends HBaseTe
     byte [] fam = Bytes.toBytes("fam");
 
     String method = this.getName();
-    this.region = initHRegion(tableName, method, fam);
+    this.region = initHRegion(tableName, method, conf, fam);
     try {
       Get get = new Get(row);
       get.addFamily(fam);
@@ -1565,7 +1558,8 @@ public class TestHRegion extends HBaseTe
   public void stestGet_Root() throws IOException {
     //Setting up region
     String method = this.getName();
-    this.region = initHRegion(HConstants.ROOT_TABLE_NAME, method, HConstants.CATALOG_FAMILY);
+    this.region = initHRegion(HConstants.ROOT_TABLE_NAME,
+      method, conf, HConstants.CATALOG_FAMILY);
     try {
       //Add to memstore
       Put put = new Put(HConstants.EMPTY_START_ROW);
@@ -1797,7 +1791,7 @@ public class TestHRegion extends HBaseTe
 
     //Setting up region
     String method = this.getName();
-    this.region = initHRegion(tableName, method, families);
+    this.region = initHRegion(tableName, method, conf, families);
     try {
       Scan scan = new Scan();
       scan.addFamily(fam1);
@@ -1822,7 +1816,7 @@ public class TestHRegion extends HBaseTe
 
     //Setting up region
     String method = this.getName();
-    this.region = initHRegion(tableName, method, families);
+    this.region = initHRegion(tableName, method, conf, families);
     try {
       Scan scan = new Scan();
       scan.addFamily(fam2);
@@ -1851,7 +1845,7 @@ public class TestHRegion extends HBaseTe
 
     //Setting up region
     String method = this.getName();
-    this.region = initHRegion(tableName, method, families);
+    this.region = initHRegion(tableName, method, conf, families);
     try {
 
       //Putting data in Region
@@ -1899,7 +1893,7 @@ public class TestHRegion extends HBaseTe
     //Setting up region
     String method = this.getName();
     try {
-      this.region = initHRegion(tableName, method, families);
+      this.region = initHRegion(tableName, method, conf, families);
     } catch (IOException e) {
       e.printStackTrace();
       fail("Got IOException during initHRegion, " + e.getMessage());
@@ -1935,7 +1929,7 @@ public class TestHRegion extends HBaseTe
 
     //Setting up region
     String method = this.getName();
-    this.region = initHRegion(tableName, method, families);
+    this.region = initHRegion(tableName, method, conf, families);
     try {
       //Putting data in Region
       Put put = null;
@@ -2002,7 +1996,7 @@ public class TestHRegion extends HBaseTe
 
     //Setting up region
     String method = this.getName();
-    this.region = initHRegion(tableName, method, families);
+    this.region = initHRegion(tableName, method, conf, families);
     try {
       //Putting data in Region
       Put put = null;
@@ -2062,7 +2056,7 @@ public class TestHRegion extends HBaseTe
 
     //Setting up region
     String method = this.getName();
-    this.region = initHRegion(tableName, method, families);
+    this.region = initHRegion(tableName, method, conf, families);
     try {
       //Putting data in Region
       Put put = null;
@@ -2127,7 +2121,7 @@ public class TestHRegion extends HBaseTe
 
     //Setting up region
     String method = this.getName();
-    this.region = initHRegion(tableName, method, families);
+    this.region = initHRegion(tableName, method, conf, families);
     try {
       //Putting data in Region
       KeyValue kv14 = new KeyValue(row1, fam1, qf1, ts4, KeyValue.Type.Put, null);
@@ -2209,7 +2203,7 @@ public class TestHRegion extends HBaseTe
 
     //Setting up region
     String method = this.getName();
-    this.region = initHRegion(tableName, method, families);
+    this.region = initHRegion(tableName, method, conf, families);
     try {
       //Putting data in Region
       Put put = null;
@@ -2270,7 +2264,7 @@ public class TestHRegion extends HBaseTe
 
     //Setting up region
     String method = this.getName();
-    this.region = initHRegion(tableName, method, fam1);
+    this.region = initHRegion(tableName, method, conf, fam1);
     try {
       //Putting data in Region
       Put put = null;
@@ -2321,7 +2315,7 @@ public class TestHRegion extends HBaseTe
   public void testScanner_StopRow1542() throws IOException {
     byte [] tableName = Bytes.toBytes("test_table");
     byte [] family = Bytes.toBytes("testFamily");
-    this.region = initHRegion(tableName, getName(), family);
+    this.region = initHRegion(tableName, getName(), conf, family);
     try {
       byte [] row1 = Bytes.toBytes("row111");
       byte [] row2 = Bytes.toBytes("row222");
@@ -2368,7 +2362,7 @@ public class TestHRegion extends HBaseTe
   }
 
   public void testIncrementColumnValue_UpdatingInPlace() throws IOException {
-    this.region = initHRegion(tableName, getName(), fam1);
+    this.region = initHRegion(tableName, getName(), conf, fam1);
     try {
       long value = 1L;
       long amount = 3L;
@@ -2396,7 +2390,7 @@ public class TestHRegion extends HBaseTe
   public void testIncrementColumnValue_BumpSnapshot() throws IOException {
     ManualEnvironmentEdge mee = new ManualEnvironmentEdge();
     EnvironmentEdgeManagerTestHelper.injectEdge(mee);
-    this.region = initHRegion(tableName, getName(), fam1);
+    this.region = initHRegion(tableName, getName(), conf, fam1);
     try {
       long value = 42L;
       long incr = 44L;
@@ -2435,7 +2429,7 @@ public class TestHRegion extends HBaseTe
   }
 
   public void testIncrementColumnValue_ConcurrentFlush() throws IOException {
-    this.region = initHRegion(tableName, getName(), fam1);
+    this.region = initHRegion(tableName, getName(), conf, fam1);
     try {
       long value = 1L;
       long amount = 3L;
@@ -2469,7 +2463,7 @@ public class TestHRegion extends HBaseTe
   public void testIncrementColumnValue_heapSize() throws IOException {
     EnvironmentEdgeManagerTestHelper.injectEdge(new IncrementingEnvironmentEdge());
 
-    this.region = initHRegion(tableName, getName(), fam1);
+    this.region = initHRegion(tableName, getName(), conf, fam1);
     try {
       long byAmount = 1L;
       long size;
@@ -2488,7 +2482,7 @@ public class TestHRegion extends HBaseTe
 
   public void testIncrementColumnValue_UpdatingInPlace_Negative()
     throws IOException {
-    this.region = initHRegion(tableName, getName(), fam1);
+    this.region = initHRegion(tableName, getName(), conf, fam1);
     try {
       long value = 3L;
       long amount = -1L;
@@ -2509,7 +2503,7 @@ public class TestHRegion extends HBaseTe
 
   public void testIncrementColumnValue_AddingNew()
     throws IOException {
-    this.region = initHRegion(tableName, getName(), fam1);
+    this.region = initHRegion(tableName, getName(), conf, fam1);
     try {
       long value = 1L;
       long amount = 3L;
@@ -2538,7 +2532,7 @@ public class TestHRegion extends HBaseTe
   }
 
   public void testIncrementColumnValue_UpdatingFromSF() throws IOException {
-    this.region = initHRegion(tableName, getName(), fam1);
+    this.region = initHRegion(tableName, getName(), conf, fam1);
     try {
       long value = 1L;
       long amount = 3L;
@@ -2566,7 +2560,7 @@ public class TestHRegion extends HBaseTe
 
   public void testIncrementColumnValue_AddingNewAfterSFCheck()
     throws IOException {
-    this.region = initHRegion(tableName, getName(), fam1);
+    this.region = initHRegion(tableName, getName(), conf, fam1);
     try {
       long value = 1L;
       long amount = 3L;
@@ -2605,7 +2599,7 @@ public class TestHRegion extends HBaseTe
    * @throws IOException
    */
   public void testIncrementColumnValue_UpdatingInPlace_TimestampClobber() throws IOException
{
-    this.region = initHRegion(tableName, getName(), fam1);
+    this.region = initHRegion(tableName, getName(), conf, fam1);
     try {
       long value = 1L;
       long amount = 3L;
@@ -2653,7 +2647,7 @@ public class TestHRegion extends HBaseTe
   }
 
   public void testIncrementColumnValue_WrongInitialSize() throws IOException {
-    this.region = initHRegion(tableName, getName(), fam1);
+    this.region = initHRegion(tableName, getName(), conf, fam1);
     try {
       byte[] row1 = Bytes.add(Bytes.toBytes("1234"), Bytes.toBytes(0L));
       int row1Field1 = 0;
@@ -2681,7 +2675,7 @@ public class TestHRegion extends HBaseTe
   }
 
   public void testIncrement_WrongInitialSize() throws IOException {
-    this.region = initHRegion(tableName, getName(), fam1);
+    this.region = initHRegion(tableName, getName(), conf, fam1);
     try {
       byte[] row1 = Bytes.add(Bytes.toBytes("1234"), Bytes.toBytes(0L));
       long row1Field1 = 0;
@@ -2757,7 +2751,7 @@ public class TestHRegion extends HBaseTe
 
     //Setting up region
     String method = this.getName();
-    this.region = initHRegion(tableName, method, fam1);
+    this.region = initHRegion(tableName, method, conf, fam1);
     try {
       //Putting data in Region
       KeyValue kv14 = new KeyValue(row1, fam1, qf1, ts4, KeyValue.Type.Put, null);
@@ -2972,7 +2966,7 @@ public class TestHRegion extends HBaseTe
     int compactInterval = 10 * flushAndScanInterval;
 
     String method = "testFlushCacheWhileScanning";
-    this.region = initHRegion(tableName,method, family);
+    this.region = initHRegion(tableName,method, conf, family);
     try {
       FlushThread flushThread = new FlushThread();
       flushThread.start();
@@ -3103,7 +3097,7 @@ public class TestHRegion extends HBaseTe
     }
 
     String method = "testWritesWhileScanning";
-    this.region = initHRegion(tableName, method, families);
+    this.region = initHRegion(tableName, method, conf, families);
     try {
       PutThread putThread = new PutThread(numRows, families, qualifiers);
       putThread.start();
@@ -3225,6 +3219,8 @@ public class TestHRegion extends HBaseTe
             }
             numPutsFinished++;
           }
+        } catch (InterruptedIOException e) {
+          // This is fine. It means we are done, or didn't get the lock on time
         } catch (IOException e) {
           LOG.error("error while putting records", e);
           error = e;
@@ -3261,8 +3257,9 @@ public class TestHRegion extends HBaseTe
       qualifiers[i] = Bytes.toBytes("qual" + i);
     }
 
+    Configuration conf = HBaseConfiguration.create(this.conf);
+
     String method = "testWritesWhileGetting";
-    Configuration conf = HBaseConfiguration.create();
     // This test flushes constantly and can cause many files to be created, possibly
     // extending over the ulimit.  Make sure compactions are aggressive in reducing
     // the number of HFiles created.
@@ -3271,7 +3268,7 @@ public class TestHRegion extends HBaseTe
     this.region = initHRegion(tableName, method, conf, families);
     PutThread putThread = null;
     MultithreadedTestUtil.TestContext ctx =
-      new MultithreadedTestUtil.TestContext(HBaseConfiguration.create());
+      new MultithreadedTestUtil.TestContext(conf);
     try {
       putThread = new PutThread(numRows, families, qualifiers);
       putThread.start();
@@ -3357,7 +3354,7 @@ public class TestHRegion extends HBaseTe
     byte[] tableName = Bytes.toBytes(method);
     byte[] family = Bytes.toBytes("family");
     this.region = initHRegion(tableName, Bytes.toBytes("x"), Bytes.toBytes("z"), method,
-        HBaseConfiguration.create(), family);
+        conf, family);
     try {
       byte[] rowNotServed = Bytes.toBytes("a");
       Get g = new Get(rowNotServed);
@@ -3421,7 +3418,7 @@ public class TestHRegion extends HBaseTe
 
     //Setting up region
     String method = "testIndexesScanWithOneDeletedRow";
-    this.region = initHRegion(tableName, method, HBaseConfiguration.create(), family);
+    this.region = initHRegion(tableName, method, conf, family);
     try {
       Put put = new Put(Bytes.toBytes(1L));
       put.add(family, qual1, 1L, Bytes.toBytes(1L));
@@ -3874,7 +3871,6 @@ public class TestHRegion extends HBaseTe
    */
   @Test
   public void testParallelIncrementWithMemStoreFlush() throws Exception {
-    Configuration conf = HBaseConfiguration.create();
     String method = "testParallelIncrementWithMemStoreFlush";
     byte[] tableName = Bytes.toBytes(method);
     byte[] family = Incrementer.family;
@@ -4007,7 +4003,8 @@ public class TestHRegion extends HBaseTe
   }
 
   private Configuration initSplit() {
-    Configuration conf = HBaseConfiguration.create();
+    Configuration conf = HBaseConfiguration.create(this.conf);
+
     // Always compact if there is more than one store file.
     conf.setInt("hbase.hstore.compactionThreshold", 2);
 
@@ -4028,19 +4025,6 @@ public class TestHRegion extends HBaseTe
   /**
    * @param tableName
    * @param callingMethod
-   * @param families
-   * @return A region on which you must call {@link HRegion#closeHRegion(HRegion)} when done.
-   * @throws IOException
-   */
-  private static HRegion initHRegion (byte [] tableName, String callingMethod,
-    byte[] ... families)
-  throws IOException {
-    return initHRegion(tableName, callingMethod, HBaseConfiguration.create(), families);
-  }
-
-  /**
-   * @param tableName
-   * @param callingMethod
    * @param conf
    * @param families
    * @throws IOException

Added: hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionBusyWait.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionBusyWait.java?rev=1417079&view=auto
==============================================================================
--- hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionBusyWait.java
(added)
+++ hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionBusyWait.java
Tue Dec  4 18:16:21 2012
@@ -0,0 +1,90 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import java.io.IOException;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.hadoop.hbase.MediumTests;
+import org.apache.hadoop.hbase.RegionTooBusyException;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+/**
+ * TestHRegion with hbase.busy.wait.duration set to 1000 (1 second).
+ * We can't use parameterized test since TestHRegion is old fashion.
+ */
+@Category(MediumTests.class)
+@SuppressWarnings("deprecation")
+public class TestHRegionBusyWait extends TestHRegion {
+  public TestHRegionBusyWait() {
+    conf.set("hbase.busy.wait.duration", "1000");
+  }
+
+  /**
+   * Test RegionTooBusyException thrown when region is busy
+   */
+  @Test (timeout=2000)
+  public void testRegionTooBusy() throws IOException {
+    String method = "testRegionTooBusy";
+    byte[] tableName = Bytes.toBytes(method);
+    byte[] family = Bytes.toBytes("family");
+    region = initHRegion(tableName, method, conf, family);
+    final AtomicBoolean stopped = new AtomicBoolean(true);
+    Thread t = new Thread(new Runnable() {
+      @Override
+      public void run() {
+        try {
+          region.lock.writeLock().lock();
+          stopped.set(false);
+          while (!stopped.get()) {
+            Thread.sleep(100);
+          }
+        } catch (InterruptedException ie) {
+        } finally {
+          region.lock.writeLock().unlock();
+        }
+      }
+    });
+    t.start();
+    Get get = new Get(row);
+    try {
+      while (stopped.get()) {
+        Thread.sleep(100);
+      }
+      region.get(get, null);
+      fail("Should throw RegionTooBusyException");
+    } catch (InterruptedException ie) {
+      fail("test interrupted");
+    } catch (RegionTooBusyException e) {
+      // Good, expected
+    } finally {
+      stopped.set(true);
+      try {
+        t.join();
+      } catch (Throwable e) {
+      }
+
+      HRegion.closeHRegion(region);
+      region = null;
+    }
+  }
+}

Propchange: hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionBusyWait.java
------------------------------------------------------------------------------
    svn:eol-style = native



Mime
View raw message