hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From st...@apache.org
Subject svn commit: r713189 - in /hadoop/hbase/trunk: ./ src/java/org/apache/hadoop/hbase/mapred/ src/java/org/apache/hadoop/hbase/master/ src/java/org/apache/hadoop/hbase/regionserver/
Date Tue, 11 Nov 2008 21:56:57 GMT
Author: stack
Date: Tue Nov 11 13:56:55 2008
New Revision: 713189

URL: http://svn.apache.org/viewvc?rev=713189&view=rev
Log:
HBASE-993 Turn of logging of every catalog table row entry on every scan

Modified:
    hadoop/hbase/trunk/CHANGES.txt
    hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/TableMapReduceUtil.java
    hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/BaseScanner.java
    hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/MetaRegion.java
    hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/MetaScanner.java
    hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/RootScanner.java
    hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/ServerManager.java
    hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java

Modified: hadoop/hbase/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/CHANGES.txt?rev=713189&r1=713188&r2=713189&view=diff
==============================================================================
--- hadoop/hbase/trunk/CHANGES.txt (original)
+++ hadoop/hbase/trunk/CHANGES.txt Tue Nov 11 13:56:55 2008
@@ -111,6 +111,7 @@
    HBASE-983   Declare Perl namespace in Hbase.thrift
    HBASE-987   We need a Hbase Partitioner for TableMapReduceUtil.initTableReduceJob
                MR Jobs (Billy Pearson via Stack)
+   HBASE-993   Turn of logging of every catalog table row entry on every scan
         
   NEW FEATURES
    HBASE-875   Use MurmurHash instead of JenkinsHash [in bloomfilters]

Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/TableMapReduceUtil.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/TableMapReduceUtil.java?rev=713189&r1=713188&r2=713189&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/TableMapReduceUtil.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/TableMapReduceUtil.java Tue
Nov 11 13:56:55 2008
@@ -102,4 +102,4 @@
       }
     }
   }
-}
+}
\ No newline at end of file

Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/BaseScanner.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/BaseScanner.java?rev=713189&r1=713188&r2=713189&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/BaseScanner.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/BaseScanner.java Tue Nov 11
13:56:55 2008
@@ -154,11 +154,11 @@
     Map<HRegionInfo, RowResult> splitParents =
       new HashMap<HRegionInfo, RowResult>();
     List<byte []> emptyRows = new ArrayList<byte []>();
+    int rows = 0;
     try {
       regionServer = master.connection.getHRegionConnection(region.getServer());
       scannerId = regionServer.openScanner(region.getRegionName(),
         COLUMN_FAMILY_ARRAY, EMPTY_START_ROW, HConstants.LATEST_TIMESTAMP, null);
-      int numberOfRegionsFound = 0;
       while (true) {
         RowResult values = regionServer.next(scannerId);
         if (values == null || values.size() == 0) {
@@ -171,20 +171,16 @@
         }
         String serverName = Writables.cellToString(values.get(COL_SERVER));
         long startCode = Writables.cellToLong(values.get(COL_STARTCODE));
-        if (LOG.isDebugEnabled()) {
-          LOG.debug(Thread.currentThread().getName() + " " + info.toString() +
-            "}, SERVER => '" + serverName + "', STARTCODE => " + startCode);
-        }
 
         // Note Region has been assigned.
         checkAssigned(info, serverName, startCode);
         if (isSplitParent(info)) {
           splitParents.put(info, values);
         }
-        numberOfRegionsFound += 1;
+        rows += 1;
       }
       if (rootRegion) {
-        regionManager.setNumMetaRegions(numberOfRegionsFound);
+        regionManager.setNumMetaRegions(rows);
       }
     } catch (IOException e) {
       if (e instanceof RemoteException) {
@@ -226,8 +222,8 @@
         cleanupSplits(region.getRegionName(), regionServer, hri, e.getValue());
       }
     }
-    LOG.info(Thread.currentThread().getName() + " scan of meta region " +
-      region.toString() + " complete");
+    LOG.info(Thread.currentThread().getName() + " scan of " + rows +
+      " row(s) of meta region " + region.toString() + " complete");
   }
 
   /*
@@ -427,4 +423,4 @@
       }
     }
   }
-}
+}
\ No newline at end of file

Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/MetaRegion.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/MetaRegion.java?rev=713189&r1=713188&r2=713189&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/MetaRegion.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/MetaRegion.java Tue Nov 11
13:56:55 2008
@@ -97,4 +97,4 @@
     }
     return result;
   }
-}
+}
\ No newline at end of file

Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/MetaScanner.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/MetaScanner.java?rev=713189&r1=713188&r2=713189&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/MetaScanner.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/MetaScanner.java Tue Nov 11
13:56:55 2008
@@ -57,15 +57,14 @@
 
   // Don't retry if we get an error while scanning. Errors are most often
   // caused by the server going away. Wait until next rescan interval when
-  // things should be back to normal
+  // things should be back to normal.
   private boolean scanOneMetaRegion(MetaRegion region) {
-    boolean scanSuccessful = false;
     while (!master.closed.get() && !regionManager.isInitialRootScanComplete() &&
       regionManager.getRootRegionLocation() == null) {
       master.sleeper.sleep();
     }
     if (master.closed.get()) {
-      return scanSuccessful;
+      return false;
     }
 
     try {
@@ -74,7 +73,6 @@
         scanRegion(region);
         regionManager.putMetaRegionOnline(region);
       }
-      scanSuccessful = true;
     } catch (IOException e) {
       e = RemoteExceptionHandler.checkIOException(e);
       LOG.warn("Scan one META region: " + region.toString(), e);
@@ -85,7 +83,7 @@
       if (!regionManager.isMetaRegionOnline(region.getStartKey())) {
         LOG.debug("Scanned region is no longer in map of online " +
         "regions or its value has changed");
-        return scanSuccessful;
+        return false;
       }
       // Make sure the file system is still available
       master.checkFileSystem();
@@ -94,7 +92,7 @@
       // at least log it rather than go out silently.
       LOG.error("Unexpected exception", e);
     }
-    return scanSuccessful;
+    return true;
   }
 
   @Override
@@ -125,22 +123,27 @@
   @Override
   protected void maintenanceScan() {
     List<MetaRegion> regions = regionManager.getListOfOnlineMetaRegions();
+    int regionCount = 0;
     for (MetaRegion r: regions) {
       scanOneMetaRegion(r);
+      regionCount++;
     }
+    LOG.info("All " + regionCount + " .META. region(s) scanned");
     metaRegionsScanned();
   }
 
-  /**
+  /*
    * Called by the meta scanner when it has completed scanning all meta 
    * regions. This wakes up any threads that were waiting for this to happen.
+   * @param totalRows Total rows scanned.
+   * @param regionCount Count of regions in  .META. table.
+   * @return False if number of meta regions matches count of online regions.
    */
   private synchronized boolean metaRegionsScanned() {
     if (!regionManager.isInitialRootScanComplete() ||
       regionManager.numMetaRegions() != regionManager.numOnlineMetaRegions()) {
       return false;
     }
-    LOG.info("all meta regions scanned");
     notifyAll();
     return true;
   }

Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/RootScanner.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/RootScanner.java?rev=713189&r1=713188&r2=713189&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/RootScanner.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/RootScanner.java Tue Nov 11
13:56:55 2008
@@ -36,14 +36,17 @@
     super(master, regionManager, true, master.metaRescanInterval, master.closed);
   }
 
-  // Don't retry if we get an error while scanning. Errors are most often
-  // caused by the server going away. Wait until next rescan interval when
-  // things should be back to normal
+  /*
+   * Don't retry if we get an error while scanning. Errors are most often
+   *
+   * caused by the server going away. Wait until next rescan interval when
+   * things should be back to normal.
+   * @return True if successfully scanned.
+   */
   private boolean scanRoot() {
-    boolean scanSuccessful = false;
     master.waitForRootRegionLocation();
     if (master.closed.get()) {
-      return scanSuccessful;
+      return false;
     }
 
     try {
@@ -54,7 +57,6 @@
             HRegionInfo.ROOT_REGIONINFO.getRegionName()));
         }
       }
-      scanSuccessful = true;
     } catch (IOException e) {
       e = RemoteExceptionHandler.checkIOException(e);
       LOG.warn("Scan ROOT region", e);
@@ -65,12 +67,12 @@
       // at least log it rather than go out silently.
       LOG.error("Unexpected exception", e);
     }
-    return scanSuccessful;
+    return true;
   }
 
   @Override
   protected boolean initialScan() {
-    initialScanComplete = scanRoot();
+    this.initialScanComplete = scanRoot();
     return initialScanComplete;
   }
 

Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/ServerManager.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/ServerManager.java?rev=713189&r1=713188&r2=713189&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/ServerManager.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/ServerManager.java Tue Nov
11 13:56:55 2008
@@ -87,7 +87,7 @@
       master.getConfiguration().getInt("hbase.master.lease.thread.wakefrequency",
         15 * 1000));
     this.loggingPeriodForAverageLoad = master.getConfiguration().
-      getLong("hbase.master.avgload.logging.period", 15000);
+      getLong("hbase.master.avgload.logging.period", 60000);
   }
   
   /**

Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java?rev=713189&r1=713188&r2=713189&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java Tue Nov
11 13:56:55 2008
@@ -369,7 +369,7 @@
         // Disable compacting and flushing by background threads for this
         // region.
         writestate.writesEnabled = false;
-        LOG.debug("Compactions and cache flushes disabled for region " + this);
+        LOG.debug("Closing " + this + ": compactions & flushes disabled ");
         while (writestate.compacting || writestate.flushing) {
           LOG.debug("waiting for" +
               (writestate.compacting ? " compaction" : "") +
@@ -384,7 +384,6 @@
         }
       }
       newScannerLock.writeLock().lock();
-      LOG.debug("Scanners disabled for region " + this);
       try {
         // Wait for active scanners to finish. The write lock we hold will
         // prevent new scanners from being created.
@@ -399,9 +398,9 @@
             }
           }
         }
-        LOG.debug("No more active scanners for region " + this);
         splitsAndClosesLock.writeLock().lock();
-        LOG.debug("Updates disabled for region " + this);
+        LOG.debug("Updates disabled for region, no outstanding scanners on " +
+          this);
         try {
           // Write lock means no more row locks can be given out.  Wait on
           // outstanding row locks to come in before we close so we do not drop
@@ -420,7 +419,7 @@
           }
           this.closed.set(true);
           
-          LOG.info("closed " + this);
+          LOG.info("Closed " + this);
           return result;
         } finally {
           splitsAndClosesLock.writeLock().unlock();



Mime
View raw message