hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From j...@apache.org
Subject svn commit: r657226 - in /hadoop/hbase/trunk: ./ src/java/org/apache/hadoop/hbase/regionserver/ src/test/org/apache/hadoop/hbase/regionserver/
Date Fri, 16 May 2008 21:45:11 GMT
Author: jimk
Date: Fri May 16 14:45:11 2008
New Revision: 657226

URL: http://svn.apache.org/viewvc?rev=657226&view=rev
Log:
HBASE-629   Split reports incorrect elapsed time

Removed:
    hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/RegionUnavailableListener.java
Modified:
    hadoop/hbase/trunk/CHANGES.txt
    hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java
    hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java
    hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
    hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestHRegion.java
    hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestSplit.java

Modified: hadoop/hbase/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/CHANGES.txt?rev=657226&r1=657225&r2=657226&view=diff
==============================================================================
--- hadoop/hbase/trunk/CHANGES.txt (original)
+++ hadoop/hbase/trunk/CHANGES.txt Fri May 16 14:45:11 2008
@@ -14,6 +14,7 @@
    HBASE-622   Remove StaticTestEnvironment and put a log4j.properties in src/test
    HBASE-624   Master will shut down if number of active region servers is zero
                even if shutdown was not requested
+   HBASE-629   Split reports incorrect elapsed time
 
   IMPROVEMENTS
    HBASE-559   MR example job to count table rows

Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java?rev=657226&r1=657225&r2=657226&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java
(original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java
Fri May 16 14:45:11 2008
@@ -41,13 +41,11 @@
 /** 
  * Compact region on request and then run split if appropriate
  */
-class CompactSplitThread extends Thread 
-implements RegionUnavailableListener, HConstants {
+class CompactSplitThread extends Thread implements HConstants {
   static final Log LOG = LogFactory.getLog(CompactSplitThread.class);
   
   private HTable root = null;
   private HTable meta = null;
-  private volatile long startTime;
   private final long frequency;
   private final ReentrantLock lock = new ReentrantLock();
   
@@ -132,7 +130,8 @@
   private void split(final HRegion region, final byte [] midKey)
   throws IOException {
     final HRegionInfo oldRegionInfo = region.getRegionInfo();
-    final HRegion[] newRegions = region.splitRegion(this, midKey);
+    final long startTime = System.currentTimeMillis();
+    final HRegion[] newRegions = region.splitRegion(midKey);
     if (newRegions == null) {
       // Didn't need to be split
       return;
@@ -190,16 +189,6 @@
     // Do not serve the new regions. Let the Master assign them.
   }
   
-  /** {@inheritDoc} */
-  public void closing(@SuppressWarnings("unused") final byte [] regionName) {
-    // continue
-  }
-  
-  /** {@inheritDoc} */
-  public void closed(@SuppressWarnings("unused") final byte [] regionName) {
-    // continue
-  }
-
   /**
    * Only interrupt once it's done with a run through the work loop.
    */ 

Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java?rev=657226&r1=657225&r2=657226&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java Fri May
16 14:45:11 2008
@@ -357,7 +357,7 @@
     new ReentrantReadWriteLock();
   private final Integer splitLock = new Integer(0);
   private final long minSequenceId;
-  private final AtomicInteger activeScannerCount = new AtomicInteger(0);
+  final AtomicInteger activeScannerCount = new AtomicInteger(0);
 
   //////////////////////////////////////////////////////////////////////////////
   // Constructor
@@ -525,7 +525,7 @@
    * @throws IOException
    */
   public List<HStoreFile> close() throws IOException {
-    return close(false, null);
+    return close(false);
   }
   
   /**
@@ -536,15 +536,13 @@
    * time-sensitive thread.
    * 
    * @param abort true if server is aborting (only during testing)
-   * @param listener call back to alert caller on close status
    * @return Vector of all the storage files that the HRegion's component 
    * HStores make use of.  It's a list of HStoreFile objects.  Can be null if
    * we are not to close at this time or we are already closed.
    * 
    * @throws IOException
    */
-  List<HStoreFile> close(boolean abort,
-      final RegionUnavailableListener listener) throws IOException {
+  List<HStoreFile> close(boolean abort) throws IOException {
     if (isClosed()) {
       LOG.warn("region " + this + " already closed");
       return null;
@@ -592,13 +590,6 @@
         waitOnRowLocks();
         LOG.debug("No more row locks outstanding on region " + this);
 
-        if (listener != null) {
-          // If there is a listener, let them know that we have now
-          // acquired all the necessary locks and are starting to
-          // do the close
-          listener.closing(getRegionName());
-        }
-        
         // Don't flush the cache if we are aborting
         if (!abort) {
           internalFlushcache();
@@ -610,12 +601,6 @@
         }
         this.closed.set(true);
         
-        if (listener != null) {
-          // If there is a listener, tell them that the region is now 
-          // closed.
-          listener.closed(getRegionName());
-        }
-        
         LOG.info("closed " + this);
         return result;
       } finally {
@@ -707,13 +692,11 @@
    * current HRegion.  Split should be fast since we don't rewrite store files
    * but instead create new 'reference' store files that read off the top and
    * bottom ranges of parent store files.
-   * @param listener May be null.
    * @param midKey key on which to split region
    * @return two brand-new (and open) HRegions or null if a split is not needed
    * @throws IOException
    */
-  HRegion[] splitRegion(final RegionUnavailableListener listener,
-      final byte [] midKey) throws IOException {
+  HRegion[] splitRegion(final byte [] midKey) throws IOException {
     synchronized (splitLock) {
       if (closed.get()) {
         return null;
@@ -753,18 +736,12 @@
       // Now close the HRegion.  Close returns all store files or null if not
       // supposed to close (? What to do in this case? Implement abort of close?)
       // Close also does wait on outstanding rows and calls a flush just-in-case.
-      List<HStoreFile> hstoreFilesToSplit = close(false, listener);
+      List<HStoreFile> hstoreFilesToSplit = close(false);
       if (hstoreFilesToSplit == null) {
         LOG.warn("Close came back null (Implement abort of close?)");
         throw new RuntimeException("close returned empty vector of HStoreFiles");
       }
 
-      // Tell listener that region is now closed and that they can therefore
-      // clean up any outstanding references.
-      if (listener != null) {
-        listener.closed(this.getRegionName());
-      }
-
       // Split each store file.
       for(HStoreFile h: hstoreFilesToSplit) {
         // A reference to the bottom half of the hsf store file.

Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java?rev=657226&r1=657225&r2=657226&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java Fri
May 16 14:45:11 2008
@@ -912,7 +912,7 @@
         LOG.debug("closing region " + Bytes.toString(region.getRegionName()));
       }
       try {
-        region.close(abortRequested, null);
+        region.close(abortRequested);
       } catch (IOException e) {
         LOG.error("error closing region " +
             Bytes.toString(region.getRegionName()),

Modified: hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestHRegion.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestHRegion.java?rev=657226&r1=657225&r2=657226&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestHRegion.java (original)
+++ hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestHRegion.java Fri
May 16 14:45:11 2008
@@ -42,10 +42,8 @@
  * A lot of the meta information for an HRegion now lives inside other
  * HRegions or in the HBaseMaster, so only basic testing is possible.
  */
-public class TestHRegion extends HBaseTestCase
-implements RegionUnavailableListener {
-  static final Logger LOG =
-    Logger.getLogger(TestHRegion.class.getName());
+public class TestHRegion extends HBaseTestCase {
+  static final Logger LOG = Logger.getLogger(TestHRegion.class);
   
   /**
    * Since all the "tests" depend on the results of the previous test, they are
@@ -60,7 +58,6 @@
       badPuts();
       basic();
       scan();
-//      batchWrite();
       splitAndMerge();
       read();
     } finally {
@@ -70,18 +67,15 @@
   
   
   private static final int FIRST_ROW = 1;
-  private static final int N_ROWS = 1000000;
   private static final int NUM_VALS = 1000;
   private static final byte [] CONTENTS_BASIC = Bytes.toBytes("contents:basic");
   private static final String CONTENTSTR = "contentstr";
   private static final String ANCHORNUM = "anchor:anchornum-";
   private static final String ANCHORSTR = "anchorstr";
-  private static final byte [] CONTENTS_BODY = Bytes.toBytes("contents:body");
   private static final byte [] CONTENTS_FIRSTCOL = Bytes.toBytes("contents:firstcol");
   private static final byte [] ANCHOR_SECONDCOL = Bytes.toBytes("anchor:secondcol");
   
   private MiniDFSCluster cluster = null;
-  private HLog log = null;
   private HTableDescriptor desc = null;
   HRegion r = null;
   HRegionIncommon region = null;
@@ -112,7 +106,6 @@
     desc.addFamily(new HColumnDescriptor("contents:"));
     desc.addFamily(new HColumnDescriptor("anchor:"));
     r = createNewHRegion(desc, null, null);
-    log = r.getLog();
     region = new HRegionIncommon(r);
     LOG.info("setup completed.");
   }
@@ -518,80 +511,13 @@
     LOG.info("scan completed.");
   }
   
-  // Do a large number of writes. Disabled if not debugging because it takes a 
-  // long time to run.
-  // Creates contents:body
-  
-  private void batchWrite() throws IOException {
-    long totalFlush = 0;
-    long totalCompact = 0;
-    long totalLog = 0;
-    long startTime = System.currentTimeMillis();
-
-    // 1M writes
-
-    int valsize = 1000;
-    for (int k = FIRST_ROW; k <= N_ROWS; k++) {
-      // Come up with a random 1000-byte string
-      String randstr1 = "" + System.currentTimeMillis();
-      StringBuffer buf1 = new StringBuffer("val_" + k + "__");
-      while (buf1.length() < valsize) {
-        buf1.append(randstr1);
-      }
-
-      // Write to the HRegion
-      BatchUpdate batchUpdate = 
-        new BatchUpdate(Bytes.toBytes("row_" + k), System.currentTimeMillis());
-      batchUpdate.put(CONTENTS_BODY,
-          buf1.toString().getBytes(HConstants.UTF8_ENCODING));
-      region.commit(batchUpdate);
-      if (k > 0 && k % (N_ROWS / 100) == 0) {
-        LOG.info("Flushing write #" + k);
-
-        long flushStart = System.currentTimeMillis();
-        region.flushcache();
-        long flushEnd = System.currentTimeMillis();
-        totalFlush += (flushEnd - flushStart);
-
-        if (k % (N_ROWS / 10) == 0) {
-          System.err.print("Rolling log...");
-          long logStart = System.currentTimeMillis();
-          log.rollWriter();
-          long logEnd = System.currentTimeMillis();
-          totalLog += (logEnd - logStart);
-          LOG.info("  elapsed time: " + ((logEnd - logStart) / 1000.0));
-        }
-      }
-    }
-    long startCompact = System.currentTimeMillis();
-    r.compactStores();
-    totalCompact = System.currentTimeMillis() - startCompact;
-    LOG.info("Region compacted - elapsedTime: " + (totalCompact / 1000.0));
-    long endTime = System.currentTimeMillis();
-
-    long totalElapsed = (endTime - startTime);
-    LOG.info("");
-    LOG.info("Batch-write complete.");
-    LOG.info("Wrote " + N_ROWS + " rows, each of ~" + valsize + " bytes");
-    LOG.info("Total flush-time: " + (totalFlush / 1000.0));
-    LOG.info("Total compact-time: " + (totalCompact / 1000.0));
-    LOG.info("Total log-time: " + (totalLog / 1000.0));
-    LOG.info("Total time elapsed: " + (totalElapsed / 1000.0));
-    LOG.info("Total time, rows/second: " + (N_ROWS / (totalElapsed / 1000.0)));
-    LOG.info("Adjusted time (not including flush, compact, or log): " + ((totalElapsed -
totalFlush - totalCompact - totalLog) / 1000.0));
-    LOG.info("Adjusted time, rows/second: " + (N_ROWS / ((totalElapsed - totalFlush - totalCompact
- totalLog) / 1000.0)));
-    LOG.info("");
-
-    LOG.info("batchWrite completed.");
-  }
-
   // NOTE: This test depends on testBatchWrite succeeding
   private void splitAndMerge() throws IOException {
     Path oldRegionPath = r.getRegionDir();
     byte [] midKey = r.compactStores();
     assertNotNull(midKey);
     long startTime = System.currentTimeMillis();
-    HRegion subregions[] = r.splitRegion(this, midKey);
+    HRegion subregions[] = r.splitRegion(midKey);
     if (subregions != null) {
       LOG.info("Split region elapsed time: "
           + ((System.currentTimeMillis() - startTime) / 1000.0));
@@ -619,20 +545,6 @@
     LOG.info("splitAndMerge completed.");
   }
 
-  /**
-   * {@inheritDoc}
-   */
-  public void closing(@SuppressWarnings("unused") final byte [] regionName) {
-    // We don't use this here. It is only for the HRegionServer
-  }
-  
-  /**
-   * {@inheritDoc}
-   */
-  public void closed(@SuppressWarnings("unused") final byte [] regionName) {
-    // We don't use this here. It is only for the HRegionServer
-  }
-  
   // This test verifies that everything is still there after splitting and merging
   
   private void read() throws IOException {

Modified: hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestSplit.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestSplit.java?rev=657226&r1=657225&r2=657226&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestSplit.java (original)
+++ hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestSplit.java Fri May
16 14:45:11 2008
@@ -207,7 +207,7 @@
   throws IOException {
     // Assert can get mid key from passed region.
     assertGet(r, COLFAMILY_NAME3, midKey);
-    HRegion [] regions = r.splitRegion(null, midKey);
+    HRegion [] regions = r.splitRegion(midKey);
     assertEquals(regions.length, 2);
     return regions;
   }



Mime
View raw message