hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From j...@apache.org
Subject svn commit: r602790 - in /lucene/hadoop/trunk/src/contrib/hbase: ./ src/java/org/apache/hadoop/hbase/ src/test/org/apache/hadoop/hbase/mapred/ src/test/org/apache/hadoop/hbase/shell/algebra/
Date Mon, 10 Dec 2007 06:41:29 GMT
Author: jimk
Date: Sun Dec  9 22:41:24 2007
New Revision: 602790

URL: http://svn.apache.org/viewvc?rev=602790&view=rev
Log:
HADOOP-2392 TestRegionServerExit has new failure mode since HADOOP-2338

Modified:
    lucene/hadoop/trunk/src/contrib/hbase/CHANGES.txt
    lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HMaster.java
    lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HStore.java
    lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/mapred/TestTableMapReduce.java
    lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/shell/algebra/TestTableJoinMapReduce.java

Modified: lucene/hadoop/trunk/src/contrib/hbase/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hbase/CHANGES.txt?rev=602790&r1=602789&r2=602790&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/contrib/hbase/CHANGES.txt (original)
+++ lucene/hadoop/trunk/src/contrib/hbase/CHANGES.txt Sun Dec  9 22:41:24 2007
@@ -103,7 +103,7 @@
    HADOOP-2333 Client side retries happen at the wrong level
    HADOOP-2357 Compaction cleanup; less deleting + prevent possible file leaks
    HADOOP-2377 Holding open MapFile.Readers is expensive, so use less of them
-
+   HADOOP-2392 TestRegionServerExit has new failure mode since HADOOP-2338
 
 Release 0.15.1
 Branch 0.15

Modified: lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HMaster.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HMaster.java?rev=602790&r1=602789&r2=602790&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HMaster.java (original)
+++ lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HMaster.java Sun
Dec  9 22:41:24 2007
@@ -1111,17 +1111,15 @@
     } catch (Throwable t) {
       LOG.fatal("Unhandled exception", t);
     }
+    // The region servers won't all exit until we stop scanning the meta regions
+    stopScanners();
+    
+    // Wait for all the remaining region servers to report in.
     letRegionServersShutdown();
 
     /*
      * Clean up and close up shop
      */
-    synchronized(rootScannerLock) {
-      rootScannerThread.interrupt();    // Wake root scanner
-    }
-    synchronized(metaScannerLock) {
-      metaScannerThread.interrupt();    // Wake meta scanner
-    }
     if (this.infoServer != null) {
       LOG.info("Stopping infoServer");
       try {
@@ -1192,6 +1190,23 @@
       LOG.error("Failed startup", e);
     }
   }
+  
+  /*
+   * Stop the root and meta scanners so that the region servers serving meta
+   * regions can shut down.
+   */
+  private void stopScanners() {
+    synchronized(rootScannerLock) {
+      if (rootScannerThread.isAlive()) {
+        rootScannerThread.interrupt();  // Wake root scanner
+      }
+    }
+    synchronized(metaScannerLock) {
+      if (metaScannerThread.isAlive()) {
+        metaScannerThread.interrupt();  // Wake meta scanner
+      }
+    }
+  }
 
   /*
    * Wait on regionservers to report in
@@ -1348,6 +1363,7 @@
           // proceed with shutdown
           LOG.info("All user tables quiesced. Proceeding with shutdown");
           closed.set(true);
+          stopScanners();
           synchronized(toDoQueue) {
             toDoQueue.clear();                         // Empty the queue
             delayedToDoQueue.clear();                  // Empty shut down queue

Modified: lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HStore.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HStore.java?rev=602790&r1=602789&r2=602790&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HStore.java (original)
+++ lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HStore.java Sun
Dec  9 22:41:24 2007
@@ -2142,12 +2142,6 @@
           }
           if (dataFilter.filterAllRemaining()) {
             moreToFollow = false;
-            LOG.debug("page limit");
-          }
-        }
-        if (LOG.isDebugEnabled()) {
-          if (this.dataFilter != null) {
-            LOG.debug("ROWKEY = " + chosenRow + ", FILTERED = " + filtered);
           }
         }
         

Modified: lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/mapred/TestTableMapReduce.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/mapred/TestTableMapReduce.java?rev=602790&r1=602789&r2=602790&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/mapred/TestTableMapReduce.java
(original)
+++ lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/mapred/TestTableMapReduce.java
Sun Dec  9 22:41:24 2007
@@ -401,9 +401,6 @@
 
         } else {
           first = new String(firstValue, HConstants.UTF8_ENCODING);
-          if (LOG.isDebugEnabled()) {
-            LOG.debug("row=" + key.getRow() + ": first value=" + first);
-          }
         }
         
         String second = "";
@@ -414,10 +411,6 @@
           fail();
           
         } else {
-          if (LOG.isDebugEnabled()) {
-            LOG.debug("row=" + key.getRow() + ": second value=" +
-                new String(secondValue, HConstants.UTF8_ENCODING));
-          }
           byte[] secondReversed = new byte[secondValue.length];
           for (int i = 0, j = secondValue.length - 1; j >= 0; j--, i++) {
             secondReversed[i] = secondValue[j];
@@ -426,7 +419,9 @@
         }
         if (first.compareTo(second) != 0) {
           if (LOG.isDebugEnabled()) {
-            LOG.debug("second key is not the reverse of first");
+            LOG.debug("second key is not the reverse of first. row=" +
+                key.getRow() + ", first value=" + first + ", second value=" +
+                second);
           }
           fail();
         }

Modified: lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/shell/algebra/TestTableJoinMapReduce.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/shell/algebra/TestTableJoinMapReduce.java?rev=602790&r1=602789&r2=602790&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/shell/algebra/TestTableJoinMapReduce.java
(original)
+++ lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/shell/algebra/TestTableJoinMapReduce.java
Sun Dec  9 22:41:24 2007
@@ -35,14 +35,19 @@
 import org.apache.hadoop.hbase.HStoreKey;
 import org.apache.hadoop.hbase.HTable;
 import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.StaticTestEnvironment;
 import org.apache.hadoop.hbase.MasterNotRunningException;
 import org.apache.hadoop.hbase.MiniHBaseCluster;
 import org.apache.hadoop.hbase.MultiRegionTable;
+import org.apache.hadoop.hbase.mapred.TableReduce;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.mapred.JobClient;
 import org.apache.hadoop.mapred.JobConf;
 import org.apache.hadoop.mapred.MiniMRCluster;
 
+/**
+ * HBase shell join test
+ */
 public class TestTableJoinMapReduce extends MultiRegionTable {
   @SuppressWarnings("hiding")
   private static final Log LOG = LogFactory.getLog(TestTableJoinMapReduce.class
@@ -64,7 +69,6 @@
   @Override
   public void setUp() throws Exception {
     super.setUp();
-    conf.setLong("hbase.hregion.max.filesize", 256 * 1024);
     dfsCluster = new MiniDFSCluster(conf, 1, true, (String[]) null);
     try {
       fs = dfsCluster.getFileSystem();
@@ -73,10 +77,7 @@
       // Start up HBase cluster
       hCluster = new MiniHBaseCluster(conf, 1, dfsCluster);
     } catch (Exception e) {
-      if (dfsCluster != null) {
-        dfsCluster.shutdown();
-        dfsCluster = null;
-      }
+      StaticTestEnvironment.shutdownDfs(dfsCluster);
       throw e;
     }
   }
@@ -90,21 +91,11 @@
     if (hCluster != null) {
       hCluster.shutdown();
     }
-
-    if (dfsCluster != null) {
-      dfsCluster.shutdown();
-    }
-
-    if (fs != null) {
-      try {
-        fs.close();
-      } catch (IOException e) {
-        LOG.info("During tear down got a " + e.getMessage());
-      }
-    }
+    StaticTestEnvironment.shutdownDfs(dfsCluster);
   }
 
   public void testTableJoinMapReduce() {
+    HTable table = null;
     try {
       HTableDescriptor desc = new HTableDescriptor(FIRST_RELATION);
       String[] columns = FIRST_COLUMNS.split(" ");
@@ -115,7 +106,7 @@
       admin.createTable(desc);
 
       // insert random data into the input table
-      HTable table = new HTable(conf, new Text(FIRST_RELATION));
+      table = new HTable(conf, new Text(FIRST_RELATION));
       for (int j = 0; j < 5; j++) {
         long lockid = table.startUpdate(new Text("rowKey" + j));
         table.put(lockid, new Text("a:"), Integer.toString(j).getBytes(
@@ -131,6 +122,10 @@
       e.printStackTrace();
     } catch (IOException e) {
       e.printStackTrace();
+    } finally {
+      if (table != null) {
+        table.close();
+      }
     }
 
     try {
@@ -143,8 +138,7 @@
       admin.createTable(desc);
 
       // insert random data into the input table
-      HTable table = new HTable(conf, new Text(SECOND_RELATION));
-
+      table = new HTable(conf, new Text(SECOND_RELATION));
       for (int j = 0; j < 3; j++) {
         long lockid = table.startUpdate(new Text("joinKey-" + j));
         table.put(lockid, new Text("d:"), ("s-" + Integer.toString(j))
@@ -158,6 +152,10 @@
       e.printStackTrace();
     } catch (IOException e) {
       e.printStackTrace();
+    } finally {
+      if (table != null) {
+        table.close();
+      }
     }
 
     try {
@@ -186,7 +184,7 @@
 
       IndexJoinMap.initJob(FIRST_RELATION, SECOND_RELATION, FIRST_COLUMNS,
           SECOND_COLUMNS, JOIN_EXPRESSION, IndexJoinMap.class, jobConf);
-      IndexJoinReduce.initJob(OUTPUT_TABLE, IndexJoinReduce.class, jobConf);
+      TableReduce.initJob(OUTPUT_TABLE, IndexJoinReduce.class, jobConf);
 
       JobClient.runJob(jobConf);
 
@@ -197,7 +195,7 @@
     }
 
     try {
-      verify(conf, OUTPUT_TABLE);
+    verify(conf, OUTPUT_TABLE);
     } catch (IOException e) {
       e.printStackTrace();
     }



Mime
View raw message