Return-Path: Delivered-To: apmail-hadoop-hbase-commits-archive@locus.apache.org Received: (qmail 65301 invoked from network); 16 May 2008 21:45:35 -0000 Received: from hermes.apache.org (HELO mail.apache.org) (140.211.11.2) by minotaur.apache.org with SMTP; 16 May 2008 21:45:35 -0000 Received: (qmail 59363 invoked by uid 500); 16 May 2008 21:45:37 -0000 Delivered-To: apmail-hadoop-hbase-commits-archive@hadoop.apache.org Received: (qmail 59346 invoked by uid 500); 16 May 2008 21:45:37 -0000 Mailing-List: contact hbase-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: hbase-dev@hadoop.apache.org Delivered-To: mailing list hbase-commits@hadoop.apache.org Received: (qmail 59337 invoked by uid 99); 16 May 2008 21:45:37 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 16 May 2008 14:45:37 -0700 X-ASF-Spam-Status: No, hits=-2000.0 required=10.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 16 May 2008 21:44:51 +0000 Received: by eris.apache.org (Postfix, from userid 65534) id 199A22388A12; Fri, 16 May 2008 14:45:12 -0700 (PDT) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r657226 - in /hadoop/hbase/trunk: ./ src/java/org/apache/hadoop/hbase/regionserver/ src/test/org/apache/hadoop/hbase/regionserver/ Date: Fri, 16 May 2008 21:45:11 -0000 To: hbase-commits@hadoop.apache.org From: jimk@apache.org X-Mailer: svnmailer-1.0.8 Message-Id: <20080516214512.199A22388A12@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: jimk Date: Fri May 16 14:45:11 2008 New Revision: 657226 URL: http://svn.apache.org/viewvc?rev=657226&view=rev Log: HBASE-629 Split reports incorrect elapsed time Removed: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/RegionUnavailableListener.java Modified: hadoop/hbase/trunk/CHANGES.txt hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestHRegion.java hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestSplit.java Modified: hadoop/hbase/trunk/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/CHANGES.txt?rev=657226&r1=657225&r2=657226&view=diff ============================================================================== --- hadoop/hbase/trunk/CHANGES.txt (original) +++ hadoop/hbase/trunk/CHANGES.txt Fri May 16 14:45:11 2008 @@ -14,6 +14,7 @@ HBASE-622 Remove StaticTestEnvironment and put a log4j.properties in src/test HBASE-624 Master will shut down if number of active region servers is zero even if shutdown was not requested + HBASE-629 Split reports incorrect elapsed time IMPROVEMENTS HBASE-559 MR example job to count table rows Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java?rev=657226&r1=657225&r2=657226&view=diff ============================================================================== --- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java (original) +++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java Fri May 16 14:45:11 2008 @@ -41,13 +41,11 @@ /** * Compact region on request and then run split if appropriate */ -class CompactSplitThread extends Thread -implements RegionUnavailableListener, HConstants { +class CompactSplitThread extends Thread implements HConstants { static final Log LOG = LogFactory.getLog(CompactSplitThread.class); private HTable root = null; private HTable meta = null; - private volatile long startTime; private final long frequency; private final ReentrantLock lock = new ReentrantLock(); @@ -132,7 +130,8 @@ private void split(final HRegion region, final byte [] midKey) throws IOException { final HRegionInfo oldRegionInfo = region.getRegionInfo(); - final HRegion[] newRegions = region.splitRegion(this, midKey); + final long startTime = System.currentTimeMillis(); + final HRegion[] newRegions = region.splitRegion(midKey); if (newRegions == null) { // Didn't need to be split return; @@ -190,16 +189,6 @@ // Do not serve the new regions. Let the Master assign them. } - /** {@inheritDoc} */ - public void closing(@SuppressWarnings("unused") final byte [] regionName) { - // continue - } - - /** {@inheritDoc} */ - public void closed(@SuppressWarnings("unused") final byte [] regionName) { - // continue - } - /** * Only interrupt once it's done with a run through the work loop. */ Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java?rev=657226&r1=657225&r2=657226&view=diff ============================================================================== --- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java (original) +++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java Fri May 16 14:45:11 2008 @@ -357,7 +357,7 @@ new ReentrantReadWriteLock(); private final Integer splitLock = new Integer(0); private final long minSequenceId; - private final AtomicInteger activeScannerCount = new AtomicInteger(0); + final AtomicInteger activeScannerCount = new AtomicInteger(0); ////////////////////////////////////////////////////////////////////////////// // Constructor @@ -525,7 +525,7 @@ * @throws IOException */ public List close() throws IOException { - return close(false, null); + return close(false); } /** @@ -536,15 +536,13 @@ * time-sensitive thread. * * @param abort true if server is aborting (only during testing) - * @param listener call back to alert caller on close status * @return Vector of all the storage files that the HRegion's component * HStores make use of. It's a list of HStoreFile objects. Can be null if * we are not to close at this time or we are already closed. * * @throws IOException */ - List close(boolean abort, - final RegionUnavailableListener listener) throws IOException { + List close(boolean abort) throws IOException { if (isClosed()) { LOG.warn("region " + this + " already closed"); return null; @@ -592,13 +590,6 @@ waitOnRowLocks(); LOG.debug("No more row locks outstanding on region " + this); - if (listener != null) { - // If there is a listener, let them know that we have now - // acquired all the necessary locks and are starting to - // do the close - listener.closing(getRegionName()); - } - // Don't flush the cache if we are aborting if (!abort) { internalFlushcache(); @@ -610,12 +601,6 @@ } this.closed.set(true); - if (listener != null) { - // If there is a listener, tell them that the region is now - // closed. - listener.closed(getRegionName()); - } - LOG.info("closed " + this); return result; } finally { @@ -707,13 +692,11 @@ * current HRegion. Split should be fast since we don't rewrite store files * but instead create new 'reference' store files that read off the top and * bottom ranges of parent store files. - * @param listener May be null. * @param midKey key on which to split region * @return two brand-new (and open) HRegions or null if a split is not needed * @throws IOException */ - HRegion[] splitRegion(final RegionUnavailableListener listener, - final byte [] midKey) throws IOException { + HRegion[] splitRegion(final byte [] midKey) throws IOException { synchronized (splitLock) { if (closed.get()) { return null; @@ -753,18 +736,12 @@ // Now close the HRegion. Close returns all store files or null if not // supposed to close (? What to do in this case? Implement abort of close?) // Close also does wait on outstanding rows and calls a flush just-in-case. - List hstoreFilesToSplit = close(false, listener); + List hstoreFilesToSplit = close(false); if (hstoreFilesToSplit == null) { LOG.warn("Close came back null (Implement abort of close?)"); throw new RuntimeException("close returned empty vector of HStoreFiles"); } - // Tell listener that region is now closed and that they can therefore - // clean up any outstanding references. - if (listener != null) { - listener.closed(this.getRegionName()); - } - // Split each store file. for(HStoreFile h: hstoreFilesToSplit) { // A reference to the bottom half of the hsf store file. Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java?rev=657226&r1=657225&r2=657226&view=diff ============================================================================== --- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java (original) +++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java Fri May 16 14:45:11 2008 @@ -912,7 +912,7 @@ LOG.debug("closing region " + Bytes.toString(region.getRegionName())); } try { - region.close(abortRequested, null); + region.close(abortRequested); } catch (IOException e) { LOG.error("error closing region " + Bytes.toString(region.getRegionName()), Modified: hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestHRegion.java URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestHRegion.java?rev=657226&r1=657225&r2=657226&view=diff ============================================================================== --- hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestHRegion.java (original) +++ hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestHRegion.java Fri May 16 14:45:11 2008 @@ -42,10 +42,8 @@ * A lot of the meta information for an HRegion now lives inside other * HRegions or in the HBaseMaster, so only basic testing is possible. */ -public class TestHRegion extends HBaseTestCase -implements RegionUnavailableListener { - static final Logger LOG = - Logger.getLogger(TestHRegion.class.getName()); +public class TestHRegion extends HBaseTestCase { + static final Logger LOG = Logger.getLogger(TestHRegion.class); /** * Since all the "tests" depend on the results of the previous test, they are @@ -60,7 +58,6 @@ badPuts(); basic(); scan(); -// batchWrite(); splitAndMerge(); read(); } finally { @@ -70,18 +67,15 @@ private static final int FIRST_ROW = 1; - private static final int N_ROWS = 1000000; private static final int NUM_VALS = 1000; private static final byte [] CONTENTS_BASIC = Bytes.toBytes("contents:basic"); private static final String CONTENTSTR = "contentstr"; private static final String ANCHORNUM = "anchor:anchornum-"; private static final String ANCHORSTR = "anchorstr"; - private static final byte [] CONTENTS_BODY = Bytes.toBytes("contents:body"); private static final byte [] CONTENTS_FIRSTCOL = Bytes.toBytes("contents:firstcol"); private static final byte [] ANCHOR_SECONDCOL = Bytes.toBytes("anchor:secondcol"); private MiniDFSCluster cluster = null; - private HLog log = null; private HTableDescriptor desc = null; HRegion r = null; HRegionIncommon region = null; @@ -112,7 +106,6 @@ desc.addFamily(new HColumnDescriptor("contents:")); desc.addFamily(new HColumnDescriptor("anchor:")); r = createNewHRegion(desc, null, null); - log = r.getLog(); region = new HRegionIncommon(r); LOG.info("setup completed."); } @@ -518,80 +511,13 @@ LOG.info("scan completed."); } - // Do a large number of writes. Disabled if not debugging because it takes a - // long time to run. - // Creates contents:body - - private void batchWrite() throws IOException { - long totalFlush = 0; - long totalCompact = 0; - long totalLog = 0; - long startTime = System.currentTimeMillis(); - - // 1M writes - - int valsize = 1000; - for (int k = FIRST_ROW; k <= N_ROWS; k++) { - // Come up with a random 1000-byte string - String randstr1 = "" + System.currentTimeMillis(); - StringBuffer buf1 = new StringBuffer("val_" + k + "__"); - while (buf1.length() < valsize) { - buf1.append(randstr1); - } - - // Write to the HRegion - BatchUpdate batchUpdate = - new BatchUpdate(Bytes.toBytes("row_" + k), System.currentTimeMillis()); - batchUpdate.put(CONTENTS_BODY, - buf1.toString().getBytes(HConstants.UTF8_ENCODING)); - region.commit(batchUpdate); - if (k > 0 && k % (N_ROWS / 100) == 0) { - LOG.info("Flushing write #" + k); - - long flushStart = System.currentTimeMillis(); - region.flushcache(); - long flushEnd = System.currentTimeMillis(); - totalFlush += (flushEnd - flushStart); - - if (k % (N_ROWS / 10) == 0) { - System.err.print("Rolling log..."); - long logStart = System.currentTimeMillis(); - log.rollWriter(); - long logEnd = System.currentTimeMillis(); - totalLog += (logEnd - logStart); - LOG.info(" elapsed time: " + ((logEnd - logStart) / 1000.0)); - } - } - } - long startCompact = System.currentTimeMillis(); - r.compactStores(); - totalCompact = System.currentTimeMillis() - startCompact; - LOG.info("Region compacted - elapsedTime: " + (totalCompact / 1000.0)); - long endTime = System.currentTimeMillis(); - - long totalElapsed = (endTime - startTime); - LOG.info(""); - LOG.info("Batch-write complete."); - LOG.info("Wrote " + N_ROWS + " rows, each of ~" + valsize + " bytes"); - LOG.info("Total flush-time: " + (totalFlush / 1000.0)); - LOG.info("Total compact-time: " + (totalCompact / 1000.0)); - LOG.info("Total log-time: " + (totalLog / 1000.0)); - LOG.info("Total time elapsed: " + (totalElapsed / 1000.0)); - LOG.info("Total time, rows/second: " + (N_ROWS / (totalElapsed / 1000.0))); - LOG.info("Adjusted time (not including flush, compact, or log): " + ((totalElapsed - totalFlush - totalCompact - totalLog) / 1000.0)); - LOG.info("Adjusted time, rows/second: " + (N_ROWS / ((totalElapsed - totalFlush - totalCompact - totalLog) / 1000.0))); - LOG.info(""); - - LOG.info("batchWrite completed."); - } - // NOTE: This test depends on testBatchWrite succeeding private void splitAndMerge() throws IOException { Path oldRegionPath = r.getRegionDir(); byte [] midKey = r.compactStores(); assertNotNull(midKey); long startTime = System.currentTimeMillis(); - HRegion subregions[] = r.splitRegion(this, midKey); + HRegion subregions[] = r.splitRegion(midKey); if (subregions != null) { LOG.info("Split region elapsed time: " + ((System.currentTimeMillis() - startTime) / 1000.0)); @@ -619,20 +545,6 @@ LOG.info("splitAndMerge completed."); } - /** - * {@inheritDoc} - */ - public void closing(@SuppressWarnings("unused") final byte [] regionName) { - // We don't use this here. It is only for the HRegionServer - } - - /** - * {@inheritDoc} - */ - public void closed(@SuppressWarnings("unused") final byte [] regionName) { - // We don't use this here. It is only for the HRegionServer - } - // This test verifies that everything is still there after splitting and merging private void read() throws IOException { Modified: hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestSplit.java URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestSplit.java?rev=657226&r1=657225&r2=657226&view=diff ============================================================================== --- hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestSplit.java (original) +++ hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestSplit.java Fri May 16 14:45:11 2008 @@ -207,7 +207,7 @@ throws IOException { // Assert can get mid key from passed region. assertGet(r, COLFAMILY_NAME3, midKey); - HRegion [] regions = r.splitRegion(null, midKey); + HRegion [] regions = r.splitRegion(midKey); assertEquals(regions.length, 2); return regions; }