hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From anto...@apache.org
Subject hbase git commit: HBASE-15135 Add metrics for storefile age
Date Mon, 22 Feb 2016 10:21:19 GMT
Repository: hbase
Updated Branches:
  refs/heads/master ed290cf82 -> e58c0385a


HBASE-15135 Add metrics for storefile age


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/e58c0385
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/e58c0385
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/e58c0385

Branch: refs/heads/master
Commit: e58c0385a738df63fa3fff287e1ddcfe6da1d046
Parents: ed290cf
Author: Mikhail Antonov <antonov@apache.org>
Authored: Mon Feb 22 02:16:40 2016 -0800
Committer: Mikhail Antonov <antonov@apache.org>
Committed: Mon Feb 22 02:21:02 2016 -0800

----------------------------------------------------------------------
 .../regionserver/MetricsRegionServerSource.java |  8 ++
 .../MetricsRegionServerWrapper.java             | 20 +++++
 .../regionserver/MetricsRegionWrapper.java      | 20 +++++
 .../MetricsRegionServerSourceImpl.java          |  8 ++
 .../regionserver/MetricsRegionSourceImpl.java   | 16 ++++
 .../TestMetricsRegionSourceImpl.java            | 20 +++++
 .../hadoop/hbase/regionserver/HStore.java       | 83 ++++++++++++++++++++
 .../MetricsRegionServerWrapperImpl.java         | 51 ++++++++++++
 .../regionserver/MetricsRegionWrapperImpl.java  | 52 ++++++++++++
 .../apache/hadoop/hbase/regionserver/Store.java | 25 ++++++
 .../hadoop/hbase/regionserver/StoreFile.java    |  7 ++
 .../hbase/regionserver/StoreFileInfo.java       | 12 +++
 .../MetricsRegionServerWrapperStub.java         | 20 +++++
 .../regionserver/MetricsRegionWrapperStub.java  | 20 +++++
 .../hbase/regionserver/TestMetricsRegion.java   | 12 +++
 .../regionserver/TestMetricsRegionServer.java   |  4 +
 .../regionserver/TestRegionServerMetrics.java   | 23 ++++++
 17 files changed, 401 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/e58c0385/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
index 0f2f90b..9b59af7 100644
--- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
+++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
@@ -155,6 +155,14 @@ public interface MetricsRegionServerSource extends BaseSource {
   String MEMSTORE_SIZE = "memStoreSize";
   String MEMSTORE_SIZE_DESC = "Size of the memstore";
   String STOREFILE_SIZE = "storeFileSize";
+  String MAX_STORE_FILE_AGE = "maxStoreFileAge";
+  String MIN_STORE_FILE_AGE = "minStoreFileAge";
+  String AVG_STORE_FILE_AGE = "avgStoreFileAge";
+  String NUM_REFERENCE_FILES = "numReferenceFiles";
+  String MAX_STORE_FILE_AGE_DESC = "Max age of store files hosted on this region server";
+  String MIN_STORE_FILE_AGE_DESC = "Min age of store files hosted on this region server";
+  String AVG_STORE_FILE_AGE_DESC = "Average age of store files hosted on this region server";
+  String NUM_REFERENCE_FILES_DESC = "Number of reference file on this region server";
   String STOREFILE_SIZE_DESC = "Size of storefiles being served.";
   String TOTAL_REQUEST_COUNT = "totalRequestCount";
   String TOTAL_REQUEST_COUNT_DESC =

http://git-wip-us.apache.org/repos/asf/hbase/blob/e58c0385/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java
b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java
index ee2b5a1..3ae6f9c 100644
--- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java
+++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java
@@ -98,6 +98,26 @@ public interface MetricsRegionServerWrapper {
   long getStoreFileSize();
 
   /**
+   * @return Max age of store files hosted on this region server
+   */
+  long getMaxStoreFileAge();
+
+  /**
+   * @return Min age of store files hosted on this region server
+   */
+  long getMinStoreFileAge();
+
+  /**
+   *  @return Average age of store files hosted on this region server
+   */
+  long getAvgStoreFileAge();
+
+  /**
+   *  @return Number of reference files on this region server
+   */
+  long getNumReferenceFiles();
+
+  /**
    * Get the number of requests per second.
    */
   double getRequestsPerSecond();

http://git-wip-us.apache.org/repos/asf/hbase/blob/e58c0385/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapper.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapper.java
b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapper.java
index 20ca9bd..a7c7096 100644
--- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapper.java
+++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapper.java
@@ -75,6 +75,26 @@ public interface MetricsRegionWrapper {
   long getFilteredReadRequestCount();
 
   /**
+   * @return Max age of store files under this region
+   */
+  long getMaxStoreFileAge();
+
+  /**
+   * @return Min age of store files under this region
+   */
+  long getMinStoreFileAge();
+
+  /**
+   *  @return Average age of store files under this region
+   */
+  long getAvgStoreFileAge();
+
+  /**
+   *  @return Number of reference files under this region
+   */
+  long getNumReferenceFiles();
+
+  /**
    * Get the total number of mutations that have been issued against this region.
    */
   long getWriteRequestCount();

http://git-wip-us.apache.org/repos/asf/hbase/blob/e58c0385/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java
b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java
index 9134f46..f669d26 100644
--- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java
+++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java
@@ -195,6 +195,14 @@ public class MetricsRegionServerSourceImpl
           .addGauge(Interns.info(STOREFILE_COUNT, STOREFILE_COUNT_DESC), rsWrap.getNumStoreFiles())
           .addGauge(Interns.info(MEMSTORE_SIZE, MEMSTORE_SIZE_DESC), rsWrap.getMemstoreSize())
           .addGauge(Interns.info(STOREFILE_SIZE, STOREFILE_SIZE_DESC), rsWrap.getStoreFileSize())
+          .addGauge(Interns.info(MAX_STORE_FILE_AGE, MAX_STORE_FILE_AGE_DESC),
+              rsWrap.getMaxStoreFileAge())
+          .addGauge(Interns.info(MIN_STORE_FILE_AGE, MIN_STORE_FILE_AGE_DESC),
+              rsWrap.getMinStoreFileAge())
+          .addGauge(Interns.info(AVG_STORE_FILE_AGE, AVG_STORE_FILE_AGE_DESC),
+              rsWrap.getAvgStoreFileAge())
+          .addGauge(Interns.info(NUM_REFERENCE_FILES, NUM_REFERENCE_FILES_DESC),
+              rsWrap.getNumReferenceFiles())
           .addGauge(Interns.info(RS_START_TIME_NAME, RS_START_TIME_DESC),
               rsWrap.getStartCode())
           .addCounter(Interns.info(TOTAL_REQUEST_COUNT, TOTAL_REQUEST_COUNT_DESC),

http://git-wip-us.apache.org/repos/asf/hbase/blob/e58c0385/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java
b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java
index fab6b51..90c6ce2 100644
--- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java
+++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java
@@ -217,6 +217,22 @@ public class MetricsRegionSourceImpl implements MetricsRegionSource {
               MetricsRegionServerSource.MEMSTORE_SIZE_DESC),
           this.regionWrapper.getMemstoreSize());
       mrb.addGauge(Interns.info(
+        regionNamePrefix + MetricsRegionServerSource.MAX_STORE_FILE_AGE,
+        MetricsRegionServerSource.MAX_STORE_FILE_AGE_DESC),
+        this.regionWrapper.getMaxStoreFileAge());
+      mrb.addGauge(Interns.info(
+        regionNamePrefix + MetricsRegionServerSource.MIN_STORE_FILE_AGE,
+        MetricsRegionServerSource.MIN_STORE_FILE_AGE_DESC),
+        this.regionWrapper.getMinStoreFileAge());
+      mrb.addGauge(Interns.info(
+        regionNamePrefix + MetricsRegionServerSource.AVG_STORE_FILE_AGE,
+        MetricsRegionServerSource.AVG_STORE_FILE_AGE_DESC),
+        this.regionWrapper.getAvgStoreFileAge());
+      mrb.addGauge(Interns.info(
+        regionNamePrefix + MetricsRegionServerSource.NUM_REFERENCE_FILES,
+        MetricsRegionServerSource.NUM_REFERENCE_FILES_DESC),
+        this.regionWrapper.getNumReferenceFiles());
+      mrb.addGauge(Interns.info(
               regionNamePrefix + MetricsRegionServerSource.STOREFILE_SIZE,
               MetricsRegionServerSource.STOREFILE_SIZE_DESC),
           this.regionWrapper.getStoreFileSize());

http://git-wip-us.apache.org/repos/asf/hbase/blob/e58c0385/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionSourceImpl.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionSourceImpl.java
b/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionSourceImpl.java
index 19624aa..4f5a8bd 100644
--- a/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionSourceImpl.java
+++ b/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionSourceImpl.java
@@ -112,6 +112,26 @@ public class TestMetricsRegionSourceImpl {
     }
 
     @Override
+    public long getMaxStoreFileAge() {
+      return 0;
+    }
+
+    @Override
+    public long getMinStoreFileAge() {
+      return 0;
+    }
+
+    @Override
+    public long getAvgStoreFileAge() {
+      return 0;
+    }
+
+    @Override
+    public long getNumReferenceFiles() {
+      return 0;
+    }
+
+    @Override
     public long getWriteRequestCount() {
       return 0;
     }

http://git-wip-us.apache.org/repos/asf/hbase/blob/e58c0385/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
index 5c29fb4..5cc3fc9 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
@@ -1888,6 +1888,89 @@ public class HStore implements Store {
   }
 
   @Override
+  public long getMaxStoreFileAge() {
+    long earliestTS = Long.MAX_VALUE;
+    for (StoreFile s: this.storeEngine.getStoreFileManager().getStorefiles()) {
+      StoreFile.Reader r = s.getReader();
+      if (r == null) {
+        LOG.warn("StoreFile " + s + " has a null Reader");
+        continue;
+      }
+      if (!s.isHFile()) {
+        continue;
+      }
+      long createdTS = s.getFileInfo().getCreatedTimestamp();
+      earliestTS = (createdTS < earliestTS) ? createdTS : earliestTS;
+    }
+    long now = EnvironmentEdgeManager.currentTime();
+    return now - earliestTS;
+  }
+
+  @Override
+  public long getMinStoreFileAge() {
+    long latestTS = 0;
+    for (StoreFile s: this.storeEngine.getStoreFileManager().getStorefiles()) {
+      StoreFile.Reader r = s.getReader();
+      if (r == null) {
+        LOG.warn("StoreFile " + s + " has a null Reader");
+        continue;
+      }
+      if (!s.isHFile()) {
+        continue;
+      }
+      long createdTS = s.getFileInfo().getCreatedTimestamp();
+      latestTS = (createdTS > latestTS) ? createdTS : latestTS;
+    }
+    long now = EnvironmentEdgeManager.currentTime();
+    return now - latestTS;
+  }
+
+  @Override
+  public long getAvgStoreFileAge() {
+    long sum = 0, count = 0;
+    for (StoreFile s: this.storeEngine.getStoreFileManager().getStorefiles()) {
+      StoreFile.Reader r = s.getReader();
+      if (r == null) {
+        LOG.warn("StoreFile " + s + " has a null Reader");
+        continue;
+      }
+      if (!s.isHFile()) {
+        continue;
+      }
+      sum += s.getFileInfo().getCreatedTimestamp();
+      count++;
+    }
+    if (count == 0) {
+      return 0;
+    }
+    long avgTS = sum / count;
+    long now = EnvironmentEdgeManager.currentTime();
+    return now - avgTS;
+  }
+
+  @Override
+  public long getNumReferenceFiles() {
+    long numRefFiles = 0;
+    for (StoreFile s : this.storeEngine.getStoreFileManager().getStorefiles()) {
+      if (s.isReference()) {
+        numRefFiles++;
+      }
+    }
+    return numRefFiles;
+  }
+
+  @Override
+  public long getNumHFiles() {
+    long numHFiles = 0;
+    for (StoreFile s : this.storeEngine.getStoreFileManager().getStorefiles()) {
+      if (s.isHFile()) {
+        numHFiles++;
+      }
+    }
+    return numHFiles;
+  }
+
+  @Override
   public long getStoreSizeUncompressed() {
     return this.totalUncompressedBytes;
   }

http://git-wip-us.apache.org/repos/asf/hbase/blob/e58c0385/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java
index 777c960..441fea6 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java
@@ -66,6 +66,10 @@ class MetricsRegionServerWrapperImpl
   private volatile long numStoreFiles = 0;
   private volatile long memstoreSize = 0;
   private volatile long storeFileSize = 0;
+  private volatile long maxStoreFileAge = 0;
+  private volatile long minStoreFileAge = 0;
+  private volatile long avgStoreFileAge = 0;
+  private volatile long numReferenceFiles = 0;
   private volatile double requestsPerSecond = 0.0;
   private volatile long readRequestsCount = 0;
   private volatile long filteredReadRequestsCount = 0;
@@ -390,6 +394,26 @@ class MetricsRegionServerWrapperImpl
   }
 
   @Override
+  public long getMaxStoreFileAge() {
+    return maxStoreFileAge;
+  }
+
+  @Override
+  public long getMinStoreFileAge() {
+    return minStoreFileAge;
+  }
+
+  @Override
+  public long getAvgStoreFileAge() {
+    return avgStoreFileAge;
+  }
+
+  @Override
+  public long getNumReferenceFiles() {
+    return numReferenceFiles;
+  }
+
+  @Override
   public long getMemstoreSize() {
     return memstoreSize;
   }
@@ -614,6 +638,9 @@ class MetricsRegionServerWrapperImpl
             new HDFSBlocksDistribution();
 
         long tempNumStores = 0, tempNumStoreFiles = 0, tempMemstoreSize = 0, tempStoreFileSize
= 0;
+        long tempMaxStoreFileAge = 0, tempNumReferenceFiles = 0;
+        long avgAgeNumerator = 0, numHFiles = 0;
+        long tempMinStoreFileAge = Long.MAX_VALUE;
         long tempReadRequestsCount = 0, tempFilteredReadRequestsCount = 0,
           tempWriteRequestsCount = 0;
         long tempCheckAndMutateChecksFailed = 0;
@@ -657,6 +684,20 @@ class MetricsRegionServerWrapperImpl
             tempNumStoreFiles += store.getStorefilesCount();
             tempMemstoreSize += store.getMemStoreSize();
             tempStoreFileSize += store.getStorefilesSize();
+
+            long storeMaxStoreFileAge = store.getMaxStoreFileAge();
+            tempMaxStoreFileAge = (storeMaxStoreFileAge > tempMaxStoreFileAge) ?
+              storeMaxStoreFileAge : tempMaxStoreFileAge;
+
+            long storeMinStoreFileAge = store.getMinStoreFileAge();
+            tempMinStoreFileAge = (storeMinStoreFileAge < tempMinStoreFileAge) ?
+              storeMinStoreFileAge : tempMinStoreFileAge;
+
+            long storeHFiles = store.getNumHFiles();
+            avgAgeNumerator += store.getAvgStoreFileAge() * storeHFiles;
+            numHFiles += storeHFiles;
+            tempNumReferenceFiles += store.getNumReferenceFiles();
+
             tempStorefileIndexSize += store.getStorefilesIndexSize();
             tempTotalStaticBloomSize += store.getTotalStaticBloomSize();
             tempTotalStaticIndexSize += store.getTotalStaticIndexSize();
@@ -723,6 +764,16 @@ class MetricsRegionServerWrapperImpl
         numStoreFiles = tempNumStoreFiles;
         memstoreSize = tempMemstoreSize;
         storeFileSize = tempStoreFileSize;
+        maxStoreFileAge = tempMaxStoreFileAge;
+        if (tempMinStoreFileAge != Long.MAX_VALUE) {
+          minStoreFileAge = tempMinStoreFileAge;
+        }
+
+        if (numHFiles != 0) {
+          avgStoreFileAge = avgAgeNumerator / numHFiles;
+        }
+
+        numReferenceFiles= tempNumReferenceFiles;
         readRequestsCount = tempReadRequestsCount;
         filteredReadRequestsCount = tempFilteredReadRequestsCount;
         writeRequestsCount = tempWriteRequestsCount;

http://git-wip-us.apache.org/repos/asf/hbase/blob/e58c0385/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperImpl.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperImpl.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperImpl.java
index 2c54079..f9e01cd 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperImpl.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperImpl.java
@@ -43,6 +43,10 @@ public class MetricsRegionWrapperImpl implements MetricsRegionWrapper,
Closeable
   private long numStoreFiles;
   private long memstoreSize;
   private long storeFileSize;
+  private long maxStoreFileAge;
+  private long minStoreFileAge;
+  private long avgStoreFileAge;
+  private long numReferenceFiles;
 
   private ScheduledFuture<?> regionMetricsUpdateTask;
 
@@ -137,6 +141,26 @@ public class MetricsRegionWrapperImpl implements MetricsRegionWrapper,
Closeable
   }
 
   @Override
+  public long getMaxStoreFileAge() {
+    return maxStoreFileAge;
+  }
+
+  @Override
+  public long getMinStoreFileAge() {
+    return minStoreFileAge;
+  }
+
+  @Override
+  public long getAvgStoreFileAge() {
+    return avgStoreFileAge;
+  }
+
+  @Override
+  public long getNumReferenceFiles() {
+    return numReferenceFiles;
+  }
+
+  @Override
   public int getRegionHashCode() {
     return this.region.hashCode();
   }
@@ -148,18 +172,46 @@ public class MetricsRegionWrapperImpl implements MetricsRegionWrapper,
Closeable
       long tempNumStoreFiles = 0;
       long tempMemstoreSize = 0;
       long tempStoreFileSize = 0;
+      long tempMaxStoreFileAge = 0;
+      long tempMinStoreFileAge = Long.MAX_VALUE;
+      long tempNumReferenceFiles = 0;
 
+      long avgAgeNumerator = 0;
+      long numHFiles = 0;
       if (region.stores != null) {
         for (Store store : region.stores.values()) {
           tempNumStoreFiles += store.getStorefilesCount();
           tempMemstoreSize += store.getMemStoreSize();
           tempStoreFileSize += store.getStorefilesSize();
+
+          long storeMaxStoreFileAge = store.getMaxStoreFileAge();
+          tempMaxStoreFileAge = (storeMaxStoreFileAge > tempMaxStoreFileAge) ?
+            storeMaxStoreFileAge : tempMaxStoreFileAge;
+
+          long storeMinStoreFileAge = store.getMinStoreFileAge();
+          tempMinStoreFileAge = (storeMinStoreFileAge < tempMinStoreFileAge) ?
+            storeMinStoreFileAge : tempMinStoreFileAge;
+
+          long storeHFiles = store.getNumHFiles();
+          avgAgeNumerator += store.getAvgStoreFileAge() * storeHFiles;
+          numHFiles += storeHFiles;
+          tempNumReferenceFiles += store.getNumReferenceFiles();
         }
       }
 
       numStoreFiles = tempNumStoreFiles;
       memstoreSize = tempMemstoreSize;
       storeFileSize = tempStoreFileSize;
+      maxStoreFileAge = tempMaxStoreFileAge;
+      if (tempMinStoreFileAge != Long.MAX_VALUE) {
+        minStoreFileAge = tempMinStoreFileAge;
+      }
+
+      if (numHFiles != 0) {
+        avgStoreFileAge = avgAgeNumerator / numHFiles;
+      }
+
+      numReferenceFiles = tempNumReferenceFiles;
     }
   }
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/e58c0385/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
index 1db213c..09e0254 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
@@ -346,6 +346,31 @@ public interface Store extends HeapSize, StoreConfigInformation, PropagatingConf
   int getStorefilesCount();
 
   /**
+   * @return Max age of store files in this store
+   */
+  long getMaxStoreFileAge();
+
+  /**
+   * @return Min age of store files in this store
+   */
+  long getMinStoreFileAge();
+
+  /**
+   *  @return Average age of store files in this store, 0 if no store files
+   */
+  long getAvgStoreFileAge();
+
+  /**
+   *  @return Number of reference files in this store
+   */
+  long getNumReferenceFiles();
+
+  /**
+   *  @return Number of HFiles in this store
+   */
+  long getNumHFiles();
+
+  /**
    * @return The size of the store files, in bytes, uncompressed.
    */
   long getStoreSizeUncompressed();

http://git-wip-us.apache.org/repos/asf/hbase/blob/e58c0385/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
index f3830ee..4ced556 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
@@ -291,6 +291,13 @@ public class StoreFile {
   }
 
   /**
+   * @return True if this is HFile.
+   */
+  public boolean isHFile() {
+    return this.fileInfo.isHFile(this.fileInfo.getPath());
+  }
+
+  /**
    * @return True if this file was made by a major compaction.
    */
   public boolean isMajorCompaction() {

http://git-wip-us.apache.org/repos/asf/hbase/blob/e58c0385/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileInfo.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileInfo.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileInfo.java
index fd7f1c6..073adae 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileInfo.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileInfo.java
@@ -97,6 +97,9 @@ public class StoreFileInfo {
 
   private RegionCoprocessorHost coprocessorHost;
 
+  // timestamp on when the file was created, is 0 and ignored for reference or link files
+  private long createdTimestamp;
+
   /**
    * Create a Store File Info
    * @param conf the {@link Configuration} to use
@@ -132,6 +135,7 @@ public class StoreFileInfo {
               " reference to " + referencePath);
     } else if (isHFile(p)) {
       // HFile
+      this.createdTimestamp = fs.getFileStatus(initialPath).getModificationTime();
       this.reference = null;
       this.link = null;
     } else {
@@ -182,6 +186,7 @@ public class StoreFileInfo {
     this.fs = fs;
     this.conf = conf;
     this.initialPath = fileStatus.getPath();
+    this.createdTimestamp = fileStatus.getModificationTime();
     this.reference = reference;
     this.link = null;
   }
@@ -419,6 +424,13 @@ public class StoreFileInfo {
     return m.matches() && m.groupCount() > 1;
   }
 
+  /**
+   * @return timestamp when this file was created (as returned by filesystem)
+   */
+  public long getCreatedTimestamp() {
+    return createdTimestamp;
+  }
+
   /*
    * Return path to the file referred to by a Reference.  Presumes a directory
    * hierarchy of <code>${hbase.rootdir}/data/${namespace}/tablename/regionname/familyname</code>.

http://git-wip-us.apache.org/repos/asf/hbase/blob/e58c0385/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java
index 95a7cf5..65c6b76 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java
@@ -71,6 +71,26 @@ public class MetricsRegionServerWrapperStub implements MetricsRegionServerWrappe
   }
 
   @Override
+  public long getMaxStoreFileAge() {
+    return 2;
+  }
+
+  @Override
+  public long getMinStoreFileAge() {
+    return 2;
+  }
+
+  @Override
+  public long getAvgStoreFileAge() {
+    return 2;
+  }
+
+  @Override
+  public long getNumReferenceFiles() {
+    return 2;
+  }
+
+  @Override
   public double getRequestsPerSecond() {
     return 0;
   }

http://git-wip-us.apache.org/repos/asf/hbase/blob/e58c0385/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperStub.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperStub.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperStub.java
index 8e6dd74..1ab44ec 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperStub.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperStub.java
@@ -81,6 +81,26 @@ public class MetricsRegionWrapperStub implements MetricsRegionWrapper {
   }
 
   @Override
+  public long getMaxStoreFileAge() {
+    return 2;
+  }
+
+  @Override
+  public long getMinStoreFileAge() {
+    return 2;
+  }
+
+  @Override
+  public long getAvgStoreFileAge() {
+    return 2;
+  }
+
+  @Override
+  public long getNumReferenceFiles() {
+    return 2;
+  }
+
+  @Override
   public long getWriteRequestCount() {
     return 106;
   }

http://git-wip-us.apache.org/repos/asf/hbase/blob/e58c0385/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegion.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegion.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegion.java
index cc09d15..12fee85 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegion.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegion.java
@@ -43,6 +43,18 @@ public class TestMetricsRegion {
       "namespace_TestNS_table_MetricsRegionWrapperStub_region_DEADBEEF001_metric_storeFileCount",
       102, agg);
     HELPER.assertGauge(
+      "namespace_TestNS_table_MetricsRegionWrapperStub_region_DEADBEEF001_metric_maxStoreFileAge",
+      2, agg);
+    HELPER.assertGauge(
+      "namespace_TestNS_table_MetricsRegionWrapperStub_region_DEADBEEF001_metric_minStoreFileAge",
+      2, agg);
+    HELPER.assertGauge(
+      "namespace_TestNS_table_MetricsRegionWrapperStub_region_DEADBEEF001_metric_avgStoreFileAge",
+      2, agg);
+    HELPER.assertGauge(
+      "namespace_TestNS_table_MetricsRegionWrapperStub_region_DEADBEEF001_metric_numReferenceFiles",
+      2, agg);
+    HELPER.assertGauge(
       "namespace_TestNS_table_MetricsRegionWrapperStub_region_DEADBEEF001_metric_memstoreSize",
       103, agg);
     HELPER.assertCounter(

http://git-wip-us.apache.org/repos/asf/hbase/blob/e58c0385/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java
index 77d6a95..4bfa64d 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java
@@ -60,6 +60,10 @@ public class TestMetricsRegionServer {
     HELPER.assertGauge("regionServerStartTime", 100, serverSource);
     HELPER.assertGauge("regionCount", 101, serverSource);
     HELPER.assertGauge("storeCount", 2, serverSource);
+    HELPER.assertGauge("maxStoreFileAge", 2, serverSource);
+    HELPER.assertGauge("minStoreFileAge", 2, serverSource);
+    HELPER.assertGauge("avgStoreFileAge", 2, serverSource);
+    HELPER.assertGauge("numReferenceFiles", 2, serverSource);
     HELPER.assertGauge("hlogFileCount", 10, serverSource);
     HELPER.assertGauge("hlogFileSize", 1024000, serverSource);
     HELPER.assertGauge("storeFileCount", 300, serverSource);

http://git-wip-us.apache.org/repos/asf/hbase/blob/e58c0385/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java
index 79df5e8..7575e7b 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java
@@ -239,6 +239,29 @@ public class TestRegionServerMetrics {
   }
 
   @Test
+  public void testStoreFileAge() throws Exception {
+    TableName tableName = TableName.valueOf("testStoreFileAge");
+    byte[] cf = Bytes.toBytes("d");
+    byte[] row = Bytes.toBytes("rk");
+    byte[] qualifier = Bytes.toBytes("qual");
+    byte[] val = Bytes.toBytes("Value");
+
+    //Force a hfile.
+    Table t = TEST_UTIL.createTable(tableName, cf);
+    Put p = new Put(row);
+    p.addColumn(cf, qualifier, val);
+    t.put(p);
+    TEST_UTIL.getHBaseAdmin().flush(tableName);
+
+    metricsRegionServer.getRegionServerWrapper().forceRecompute();
+    assertTrue(metricsHelper.getGaugeLong("maxStoreFileAge", serverSource) > 0);
+    assertTrue(metricsHelper.getGaugeLong("minStoreFileAge", serverSource) > 0);
+    assertTrue(metricsHelper.getGaugeLong("avgStoreFileAge", serverSource) > 0);
+
+    t.close();
+  }
+
+  @Test
   public void testCheckAndPutCount() throws Exception {
     String tableNameString = "testCheckAndPutCount";
     TableName tableName = TableName.valueOf(tableNameString);


Mime
View raw message