hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From st...@apache.org
Subject svn commit: r1344447 - in /hbase/branches/0.90: ./ src/main/java/org/apache/hadoop/hbase/client/ src/main/java/org/apache/hadoop/hbase/ipc/ src/main/java/org/apache/hadoop/hbase/regionserver/ src/main/resources/hbase-webapps/master/
Date Wed, 30 May 2012 20:23:49 GMT
Author: stack
Date: Wed May 30 20:23:49 2012
New Revision: 1344447

URL: http://svn.apache.org/viewvc?rev=1344447&view=rev
Log:
HBASE-6124 Backport HBASE-6033 to 0.90, 0.92 and 0.94

Modified:
    hbase/branches/0.90/CHANGES.txt
    hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
    hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/ipc/HRegionInterface.java
    hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java
    hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
    hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
    hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/regionserver/PriorityCompactionQueue.java
    hbase/branches/0.90/src/main/resources/hbase-webapps/master/table.jsp

Modified: hbase/branches/0.90/CHANGES.txt
URL: http://svn.apache.org/viewvc/hbase/branches/0.90/CHANGES.txt?rev=1344447&r1=1344446&r2=1344447&view=diff
==============================================================================
--- hbase/branches/0.90/CHANGES.txt (original)
+++ hbase/branches/0.90/CHANGES.txt Wed May 30 20:23:49 2012
@@ -32,6 +32,7 @@ Release 0.90.7 - Unreleased
    HBASE-5922  HalfStoreFileReader seekBefore causes StackOverflowError (Nate Putnam)
    HBASE-5757  TableInputFormat should handle as many errors as possible (Jan Lukavsky)
    HBASE-6041  NullPointerException prevents the master from starting up (Jimmy Xiang)
+   HBASE-6124  https://issues.apache.org/jira/secure/attachment/12530072/patch-0.90.txt (Jimmy
Xiang)
 
   IMPROVEMENT
    HBASE-5588  Deprecate/remove AssignmentManager#clearRegionFromTransition

Modified: hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java?rev=1344447&r1=1344446&r2=1344447&view=diff
==============================================================================
--- hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java (original)
+++ hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java Wed May
30 20:23:49 2012
@@ -53,6 +53,7 @@ import org.apache.hadoop.hbase.catalog.M
 import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor;
 import org.apache.hadoop.hbase.ipc.HMasterInterface;
 import org.apache.hadoop.hbase.ipc.HRegionInterface;
+import org.apache.hadoop.hbase.regionserver.CompactSplitThread.CompactionState;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.Pair;
 import org.apache.hadoop.hbase.util.Writables;
@@ -1349,4 +1350,88 @@ public class HBaseAdmin implements Abort
       this.connection.close();
     }
   }
+  /**
+   * Get the current compaction state of a table or region.
+   * It could be in a major compaction, a minor compaction, both, or none.
+   *
+   * @param tableNameOrRegionName table or region to major compact
+   * @throws IOException if a remote or network exception occurs
+   * @throws InterruptedException
+   * @return the current compaction state
+   */
+  public CompactionState getCompactionState(final String tableNameOrRegionName)
+      throws IOException, InterruptedException {
+    return getCompactionState(Bytes.toBytes(tableNameOrRegionName));
+  }
+
+  /**
+   * Get the current compaction state of a table or region.
+   * It could be in a major compaction, a minor compaction, both, or none.
+   *
+   * @param tableNameOrRegionName table or region to major compact
+   * @throws IOException if a remote or network exception occurs
+   * @throws InterruptedException
+   * @return the current compaction state
+   */
+  public CompactionState getCompactionState(final byte [] tableNameOrRegionName)
+      throws IOException, InterruptedException {
+    CompactionState state = CompactionState.NONE;
+    CatalogTracker ct = getCatalogTracker();
+    try {
+      if (isRegionName(tableNameOrRegionName, ct)) {
+        Pair<HRegionInfo, HServerAddress> pair =
+          MetaReader.getRegion(ct, tableNameOrRegionName);
+        if (pair == null || pair.getSecond() == null) {
+          LOG.info("No server in .META. for " +
+            Bytes.toStringBinary(tableNameOrRegionName) + "; pair=" + pair);
+        } else {
+          HRegionInterface rs =
+            this.connection.getHRegionConnection(pair.getSecond());
+          return CompactionState.valueOf(
+            rs.getCompactionState(pair.getFirst().getRegionName()));
+        }
+      } else {
+        final String tableName = tableNameString(tableNameOrRegionName, ct);
+        List<Pair<HRegionInfo, HServerAddress>> pairs =
+          MetaReader.getTableRegionsAndLocations(ct, tableName);
+        for (Pair<HRegionInfo, HServerAddress> pair: pairs) {
+          if (pair.getFirst().isOffline()) continue;
+          if (pair.getSecond() == null) continue;
+          try {
+            HRegionInterface rs =
+              this.connection.getHRegionConnection(pair.getSecond());
+            switch (CompactionState.valueOf(
+              rs.getCompactionState(pair.getFirst().getRegionName()))) {
+            case MAJOR_AND_MINOR:
+              return CompactionState.MAJOR_AND_MINOR;
+            case MAJOR:
+              if (state == CompactionState.MINOR) {
+                return CompactionState.MAJOR_AND_MINOR;
+              }
+              state = CompactionState.MAJOR;
+              break;
+            case MINOR:
+              if (state == CompactionState.MAJOR) {
+                return CompactionState.MAJOR_AND_MINOR;
+              }
+              state = CompactionState.MINOR;
+              break;
+            case NONE:
+              default:
+                // nothing, continue
+            }
+          } catch (NotServingRegionException e) {
+            if (LOG.isDebugEnabled()) {
+              LOG.debug("Trying to get compaction state of " +
+                pair.getFirst() + ": " +
+                StringUtils.stringifyException(e));
+            }
+          }
+        }
+      }
+    } finally {
+      cleanupCatalogTracker(ct);
+    }
+    return state;
+  }
 }

Modified: hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/ipc/HRegionInterface.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/ipc/HRegionInterface.java?rev=1344447&r1=1344446&r2=1344447&view=diff
==============================================================================
--- hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/ipc/HRegionInterface.java (original)
+++ hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/ipc/HRegionInterface.java Wed
May 30 20:23:49 2012
@@ -433,4 +433,13 @@ public interface HRegionInterface extend
   /* TODO: Move into place above master operations after deprecation cycle */
   public boolean bulkLoadHFiles(List<Pair<byte[], String>> familyPaths, byte[]
regionName)
   throws IOException;
+
+  /**
+   * Get the current compaction state of the region.
+   *
+   * @param regionName the name of the region to check compaction statte.
+   * @return the compaction state name.
+   * @throws IOException exception
+   */
+  public String getCompactionState(final byte[] regionName) throws IOException;
 }

Modified: hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java?rev=1344447&r1=1344446&r2=1344447&view=diff
==============================================================================
--- hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java
(original)
+++ hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java
Wed May 30 20:23:49 2012
@@ -20,6 +20,7 @@
 package org.apache.hadoop.hbase.regionserver;
 
 import java.io.IOException;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.locks.ReentrantLock;
 
@@ -49,6 +50,53 @@ public class CompactSplitThread extends 
   public static final int PRIORITY_USER = 1;
 
   /**
+   * Map to track the state of compactions requested per region (id)
+   */
+  private static final ConcurrentHashMap<Long, CompactionState>
+    compactionStates = new ConcurrentHashMap<Long, CompactionState>();
+
+  /**
+   * Find out if a given region is in compaction now.
+   * This information is not accurate in case one request is
+   * being processed while new request comes. So it is just
+   * an indication for normal scenario.
+   *
+   * @param regionId
+   * @return
+   */
+  public static CompactionState getCompactionState(
+      final long regionId) {
+    Long key = Long.valueOf(regionId);
+    CompactionState state = compactionStates.get(key);
+    if (state == null) {
+      state = CompactionState.NONE;
+    }
+    return state;
+  }
+
+  public static void preRequest(final HRegion region){
+    Long key = Long.valueOf(region.getRegionId());
+    CompactionState state = 
+      region.getForceMajorCompaction() ? CompactionState.MAJOR : CompactionState.MINOR;
+    compactionStates.put(key, state); 
+  }
+
+  public static void postRequest(final HRegion region){
+    Long key = Long.valueOf(region.getRegionId());
+    compactionStates.remove(key);
+  }
+
+  /**
+   * An enum for the region compaction state
+   */
+  public static enum CompactionState {
+    NONE,
+    MINOR,
+    MAJOR,
+    MAJOR_AND_MINOR;
+  }
+
+  /**
    * Splitting should not take place if the total number of regions exceed this.
    * This is not a hard limit to the number of regions but it is a guideline to
    * stop splitting after number of online regions is greater than this.
@@ -89,6 +137,7 @@ public class CompactSplitThread extends 
             }
           } finally {
             lock.unlock();
+            CompactSplitThread.postRequest(r);
           }
         }
       } catch (InterruptedException ex) {

Modified: hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java?rev=1344447&r1=1344446&r2=1344447&view=diff
==============================================================================
--- hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java (original)
+++ hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java Wed
May 30 20:23:49 2012
@@ -2045,6 +2045,32 @@ public class HRegion implements HeapSize
     return this.stores.get(column);
   }
 
+  /**
+   * Return list of storeFiles for the set of CFs.
+   * Uses closeLock to prevent the race condition where a region closes
+   * in between the for loop - closing the stores one by one, some stores
+   * will return 0 files.
+   * @return List of storeFiles.
+   */
+  public List<String> getStoreFileList(final byte [][] columns)
+    throws IllegalArgumentException {
+    List<String> storeFileNames = new ArrayList<String>();
+    synchronized(closeLock) {
+      for(byte[] column : columns) {
+        Store store = this.stores.get(column);
+        if (store == null) {
+          throw new IllegalArgumentException("No column family : " +
+              new String(column) + " available");
+        }
+        List<StoreFile> storeFiles = store.getStorefiles();
+        for (StoreFile storeFile: storeFiles) {
+          storeFileNames.add(storeFile.getPath().toString());
+        }
+      }
+    }
+    return storeFileNames;
+  }
+
   //////////////////////////////////////////////////////////////////////////////
   // Support code
   //////////////////////////////////////////////////////////////////////////////

Modified: hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java?rev=1344447&r1=1344446&r2=1344447&view=diff
==============================================================================
--- hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
(original)
+++ hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
Wed May 30 20:23:49 2012
@@ -2798,4 +2798,42 @@ public class HRegionServer implements HR
 
     new HRegionServerCommandLine(regionServerClass).doMain(args);
   }
+
+  /**
+   * Gets the online regions of the specified table.
+   * This method looks at the in-memory onlineRegions.  It does not go to <code>.META.</code>.
+   * Only returns <em>online</em> regions.  If a region on this table has been
+   * closed during a disable, etc., it will not be included in the returned list.
+   * So, the returned list may not necessarily be ALL regions in this table, its
+   * all the ONLINE regions in the table.
+   * @param tableName
+   * @return Online regions from <code>tableName</code>
+   */
+   public List<HRegion> getOnlineRegions(byte[] tableName) {
+     List<HRegion> tableRegions = new ArrayList<HRegion>();
+     synchronized (this.onlineRegions) {
+       for (HRegion region: this.onlineRegions.values()) {
+         HRegionInfo regionInfo = region.getRegionInfo();
+         if(Bytes.equals(regionInfo.getTableName(), tableName)) {
+           tableRegions.add(region);
+         }
+       }
+     }
+     return tableRegions;
+   }
+
+  /**
+   * Get the current compaction state of the region.
+   *
+   * @param regionName the name of the region to check compaction statte.
+   * @return the compaction state name.
+   * @throws IOException exception
+   */
+  public String getCompactionState(final byte[] regionName) throws IOException {
+      checkOpen();
+      requestCount.incrementAndGet();
+      HRegion region = getRegion(regionName);
+      HRegionInfo info = region.getRegionInfo();
+      return CompactSplitThread.getCompactionState(info.getRegionId()).name();
+  }
 }

Modified: hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/regionserver/PriorityCompactionQueue.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/regionserver/PriorityCompactionQueue.java?rev=1344447&r1=1344446&r2=1344447&view=diff
==============================================================================
--- hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/regionserver/PriorityCompactionQueue.java
(original)
+++ hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/regionserver/PriorityCompactionQueue.java
Wed May 30 20:23:49 2012
@@ -20,7 +20,6 @@
 package org.apache.hadoop.hbase.regionserver;
 
 import java.util.Collection;
-import java.util.Date;
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.concurrent.BlockingQueue;
@@ -44,7 +43,7 @@ public class PriorityCompactionQueue imp
    * This class represents a compaction request and holds the region, priority,
    * and time submitted.
    */
-  private class CompactionRequest implements Comparable<CompactionRequest> {
+  static class CompactionRequest implements Comparable<CompactionRequest> {
     private final HRegion r;
     private final int p;
     private final Long timeInNanos;
@@ -144,6 +143,7 @@ public class PriorityCompactionQueue imp
           newRequest.getPriority() < queuedRequest.getPriority()) {
         LOG.trace("Inserting region in queue. " + newRequest);
         regionsInQueue.put(r, newRequest);
+        CompactSplitThread.preRequest(r);
       } else {
         LOG.trace("Region already in queue, skipping. Queued: " + queuedRequest +
           ", requested: " + newRequest);
@@ -187,8 +187,7 @@ public class PriorityCompactionQueue imp
   public boolean add(HRegion e, int p) {
     CompactionRequest request = this.addToRegionsInQueue(e, p);
     if (request != null) {
-      boolean result = queue.add(request);
-      return result;
+      return queue.add(request);
     } else {
       return false;
     }

Modified: hbase/branches/0.90/src/main/resources/hbase-webapps/master/table.jsp
URL: http://svn.apache.org/viewvc/hbase/branches/0.90/src/main/resources/hbase-webapps/master/table.jsp?rev=1344447&r1=1344446&r2=1344447&view=diff
==============================================================================
--- hbase/branches/0.90/src/main/resources/hbase-webapps/master/table.jsp (original)
+++ hbase/branches/0.90/src/main/resources/hbase-webapps/master/table.jsp Wed May 30 20:23:49
2012
@@ -146,6 +146,11 @@
       <td><%= hbadmin.isTableEnabled(table.getTableName()) %></td>
       <td>Is the table enabled</td>
   </tr>
+  <tr>
+      <td>Compaction</td>
+      <td><%= hbadmin.getCompactionState(table.getTableName()) %></td>
+      <td>Is the table compacting</td>
+  </tr>
 <%  if (showFragmentation) { %>
   <tr>
       <td>Fragmentation</td>



Mime
View raw message