hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From li...@apache.org
Subject svn commit: r1470790 - in /hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase: io/hfile/AbstractHFileWriter.java io/hfile/HFile.java io/hfile/HFileWriterV2.java regionserver/Store.java regionserver/metrics/RegionServerMetrics.java
Date Tue, 23 Apr 2013 02:49:32 GMT
Author: liyin
Date: Tue Apr 23 02:49:31 2013
New Revision: 1470790

URL: http://svn.apache.org/r1470790
Log:
[HBASE-5263] Change the logging to trace in HFileWriterV2 for cacheCurrentBlockForCompaction

Author: rshroff

Summary:
The logging level is debug by default on all the HBase clusters. The current
function cacheCurrentBlockForCompaction dumps the log at debug level for every
new block written during compaction.

Changing the level to trace to avoid spamming the log.

Test Plan: build and test it on TSH25

Reviewers: liyintang, nspiegelberg

Reviewed By: liyintang

CC: hbase-eng@

Differential Revision: https://phabricator.fb.com/D772585

Task ID: 2016630

Modified:
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/AbstractHFileWriter.java
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java

Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/AbstractHFileWriter.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/AbstractHFileWriter.java?rev=1470790&r1=1470789&r2=1470790&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/AbstractHFileWriter.java
(original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/AbstractHFileWriter.java
Tue Apr 23 02:49:31 2013
@@ -127,6 +127,13 @@ public abstract class AbstractHFileWrite
 
   protected boolean isCompactionWriter = false;
 
+  protected int numBlocksCachedPerCompaction = 0;
+
+  @Override
+  public int getNumBlocksCachedPerCompaction() {
+    return numBlocksCachedPerCompaction;
+  }
+
   public AbstractHFileWriter(Configuration conf, CacheConfig cacheConf,
       FSDataOutputStream outputStream, Path path, int blockSize,
       Compression.Algorithm compressAlgo,

Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java?rev=1470790&r1=1470789&r2=1470790&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java Tue Apr
23 02:49:31 2013
@@ -164,6 +164,7 @@ public class HFile {
 
   static final AtomicInteger writeOps = new AtomicInteger();
   static final AtomicLong writeTimeNano = new AtomicLong();
+  static final AtomicInteger blocksCachedDuringCompaction = new AtomicInteger();
 
   // For measuring latency of pread
   static final AtomicInteger preadOps = new AtomicInteger();
@@ -190,6 +191,13 @@ public class HFile {
   }
 
   /**
+   * Get number of blocks cached during compaction and reset the counter
+   */
+  public static final int getBlocksCachedDuringCompactionAndReset() {
+    return blocksCachedDuringCompaction.getAndSet(0);
+  }
+
+  /**
    * Get the total time of positional reads during compaction in milliseconds and reset it
to zero.
    */
   public static final long getPreadCompactionTimeMsAndReset() {
@@ -277,6 +285,11 @@ public class HFile {
      * Set whether compaction is in progress or not
      */
     void setCompactionWriter(boolean isCompaction);
+
+    /**
+     * Returns the number of blocks cached during compaction
+     */
+    int getNumBlocksCachedPerCompaction();
   }
 
   /**

Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java?rev=1470790&r1=1470789&r2=1470790&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java
(original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java
Tue Apr 23 02:49:31 2013
@@ -176,13 +176,16 @@ public class HFileWriterV2 extends Abstr
     float blockCachingPercentage =
         ((float)this.numCachedKeysInCurrentBlock)/this.numKeysInCurrentBlock;
 
-    LOG.debug("Block Caching %: " + blockCachingPercentage +
-        " CachingOnCompaction: " + this.cacheConf.shouldCacheOnCompaction() +
-        " Threshold: " + this.cacheConf.getCacheOnCompactionThreshold());
-
+    if (LOG.isTraceEnabled()) {
+      LOG.trace("Block Caching %: " + blockCachingPercentage +
+          " CachingOnCompaction: " + this.cacheConf.shouldCacheOnCompaction() +
+          " Threshold: " + this.cacheConf.getCacheOnCompactionThreshold());
+    }
     // Get the threshold from the config
     if (this.cacheConf.shouldCacheOnCompaction() &&
         blockCachingPercentage >= this.cacheConf.getCacheOnCompactionThreshold())  {
+      this.numBlocksCachedPerCompaction++;
+      HFile.blocksCachedDuringCompaction.incrementAndGet();
       return true;
     }
     return false;

Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java?rev=1470790&r1=1470789&r2=1470790&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java Tue
Apr 23 02:49:31 2013
@@ -965,6 +965,10 @@ public class Store extends SchemaConfigu
           + ", size=" + (sf == null? "none" : StringUtils.humanReadableInt(sf.getReader().length()))
           + "; total size for store is "
           + StringUtils.humanReadableInt(storeSize));
+      if (writer != null) {
+        LOG.info(", number of blocks precached="
+            + writer.getHFileWriter().getNumBlocksCachedPerCompaction());
+      }
     } catch (IOException ioe) {
       // rather than leak the status, we abort here, then rethrow the exception
       status.abort(StringUtils.stringifyException(ioe));

Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java?rev=1470790&r1=1470789&r2=1470790&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java
(original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java
Tue Apr 23 02:49:31 2013
@@ -176,6 +176,12 @@ public class RegionServerMetrics impleme
       new MetricsTimeVaryingRate("fsReadLatency", registry);
 
   /**
+   * number of blocks cached during compaction
+   */
+  public final MetricsIntValue blocksCachedDuringCompaction =
+      new MetricsIntValue("blocksCachedDuringCompaction", registry);
+
+  /**
    * filesystem p99 read latency outlier for positional read operations
    */
   public final PercentileMetric fsReadLatencyP99 =
@@ -424,6 +430,9 @@ public class RegionServerMetrics impleme
         this.rowLockTime.inc(writeOps, HRegion.getRowLockTime());
       }
 
+      this.blocksCachedDuringCompaction.set(
+          HFile.getBlocksCachedDuringCompactionAndReset());
+
       // push the result
       this.fsReadLatencyP99.pushMetric(this.metricsRecord);
       this.fsCompactionReadLatencyP99.pushMetric(this.metricsRecord);
@@ -454,6 +463,7 @@ public class RegionServerMetrics impleme
       this.quorumReadWins.pushMetric(this.metricsRecord);
       this.quorumReadsExecutedInCurThread.pushMetric(this.metricsRecord);
 
+      this.blocksCachedDuringCompaction.pushMetric(this.metricsRecord);
     }
     this.metricsRecord.update();
   }



Mime
View raw message