hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From li...@apache.org
Subject svn commit: r1584857 - in /hbase/branches/0.89-fb/src: main/java/org/apache/hadoop/hbase/client/ main/java/org/apache/hadoop/hbase/ipc/ main/java/org/apache/hadoop/hbase/ipc/thrift/ main/java/org/apache/hadoop/hbase/regionserver/ test/java/org/apache/h...
Date Fri, 04 Apr 2014 18:18:44 GMT
Author: liyin
Date: Fri Apr  4 18:18:43 2014
New Revision: 1584857

URL: http://svn.apache.org/r1584857
Log:
[HBASE-9815] Provide a batch API to retrieve Histograms for a list of regions in a regionserver.

Author: manukranthk

Summary: This diff attempts to add a HRegionServer API which gets a list of histograms for
the given list of regions.

Test Plan: Modified the unit test TestHFileHistogramE2E to add checks to compare the return
from the batch function with the previous one.

Reviewers: liyintang, aaiyer

Reviewed By: liyintang

CC: hbase-eng@, andrewcox

Differential Revision: https://phabricator.fb.com/D1256318

Task ID: 3875260

Modified:
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/HTable.java
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/ipc/HRegionInterface.java
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/ipc/ThriftHRegionInterface.java
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/ipc/thrift/HBaseToThriftAdapter.java
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/ThriftHRegionServer.java
    hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestHFileHistogramE2E.java

Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/HTable.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/HTable.java?rev=1584857&r1=1584856&r2=1584857&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/HTable.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/HTable.java Fri Apr
 4 18:18:43 2014
@@ -27,6 +27,7 @@ import java.util.Arrays;
 import java.util.Collection;
 import java.util.List;
 import java.util.Map;
+import java.util.Map.Entry;
 import java.util.NavigableMap;
 import java.util.TreeMap;
 import java.util.concurrent.Callable;
@@ -1414,6 +1415,17 @@ getRegionCachePrefetch(new StringBytes(t
   }
 
   /**
+   * Returns all the histograms related to the current table.
+   * This is a batch operation including nultiple
+   * @return
+   * @throws IOException
+   */
+  public List<List<Bucket>> batchgetHistogramsForAllRegions()
+      throws IOException {
+    return getHistogramForAllRegionsInternal();
+  }
+
+  /**
    * API to get the histograms for all the regions in the table.
    * @param family : the family whose histogram is being requested.
    * Returns data for all the families if family is null.
@@ -1452,6 +1464,57 @@ getRegionCachePrefetch(new StringBytes(t
     return ret;
   }
 
+  public Map<HServerAddress, List<HRegionInfo>> getServersToRegionsMap()
+      throws IOException {
+    Map<HServerAddress, List<HRegionInfo>> serversToRegionsMap =
+        new TreeMap<>();
+    for (Entry<HRegionInfo, HServerAddress> entry : getRegionsInfo().entrySet())
+    {
+      List<HRegionInfo> lst = serversToRegionsMap.get(entry.getValue());
+      if (lst == null) {
+        lst = new ArrayList<>();
+      }
+      lst.add(entry.getKey());
+      serversToRegionsMap.put(entry.getValue(), lst);
+    }
+    return serversToRegionsMap;
+  }
+
+  private List<List<Bucket>> getHistogramForAllRegionsInternal()
+      throws IOException {
+    List<List<Bucket>> ret = new ArrayList<List<Bucket>>();
+    TreeMap<byte[], List<Bucket>> retMap =
+        new TreeMap<>(Bytes.BYTES_COMPARATOR);
+    List<Future<List<List<Bucket>>>>futures = new ArrayList<>();
+    for (final Entry<HServerAddress, List<HRegionInfo>> entry :
+      getServersToRegionsMap().entrySet()) {
+      final List<byte[]> regionNames = new ArrayList<>();
+      for (HRegionInfo info : entry.getValue()) {
+        regionNames.add(info.getRegionName());
+      }
+      futures.add(HTable.multiActionThreadPool.submit(new Callable<List<List<Bucket>>>()
{
+        @Override
+        public List<List<Bucket>> call() throws Exception {
+          return getConnectionAndResetOperationContext()
+                .getHRegionConnection(entry.getKey()).getHistograms(regionNames);
+        }
+      }));
+    }
+    for (Future<List<List<Bucket>>> future : futures) {
+      try {
+        for (List<Bucket> histogram : future.get()) {
+          if (histogram.size() >= 1) {
+            retMap.put(histogram.get(0).getStartRow(), histogram);
+          }
+        }
+      } catch (InterruptedException | ExecutionException e) {
+        throw new IOException(e);
+      }
+    }
+    ret.addAll(retMap.values());
+    return ret;
+  }
+
   public long getMaxScannerResultSize() {
     return maxScannerResultSize;
   }

Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/ipc/HRegionInterface.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/ipc/HRegionInterface.java?rev=1584857&r1=1584856&r2=1584857&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/ipc/HRegionInterface.java
(original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/ipc/HRegionInterface.java
Fri Apr  4 18:18:43 2014
@@ -472,4 +472,14 @@ public interface HRegionInterface extend
    */
   public List<Bucket> getHistogramForStore(byte[] regionName, byte[] family)
       throws IOException;
+
+  /**
+   * Returns the list of buckets which represent the uniform depth histogram
+   * for all the given regions
+   * @param regionName
+   * @return
+   * @throws IOException
+   */
+  public List<List<Bucket>> getHistograms(List<byte[]> regionNames)
+      throws IOException;
 }

Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/ipc/ThriftHRegionInterface.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/ipc/ThriftHRegionInterface.java?rev=1584857&r1=1584856&r2=1584857&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/ipc/ThriftHRegionInterface.java
(original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/ipc/ThriftHRegionInterface.java
Fri Apr  4 18:18:43 2014
@@ -655,4 +655,9 @@ public interface ThriftHRegionInterface 
       @ThriftException(type = ThriftHBaseException.class, id = 1) })
   public List<Bucket> getHistogramForStore(byte[] regionName, byte[] family)
       throws ThriftHBaseException;
+
+  @ThriftMethod(value = "getHistograms", exception = {
+      @ThriftException(type = ThriftHBaseException.class, id = 1) })
+  public List<List<Bucket>> getHistograms(List<byte[]> regionNames)
+      throws ThriftHBaseException;
 }

Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/ipc/thrift/HBaseToThriftAdapter.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/ipc/thrift/HBaseToThriftAdapter.java?rev=1584857&r1=1584856&r2=1584857&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/ipc/thrift/HBaseToThriftAdapter.java
(original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/ipc/thrift/HBaseToThriftAdapter.java
Fri Apr  4 18:18:43 2014
@@ -1206,4 +1206,23 @@ public class HBaseToThriftAdapter implem
     }
   }
 
+  @Override
+  public List<List<Bucket>> getHistograms(List<byte[]> regionNames)
+      throws IOException {
+    preProcess();
+    try {
+      return connection.getHistograms(regionNames);
+    } catch (ThriftHBaseException te) {
+      Exception e = te.getServerJavaException();
+      handleIOException(e);
+      LOG.warn("Unexpected Exception: " + e);
+      return null;
+    } catch (Exception e) {
+      refreshConnectionAndThrowIOException(e);
+      return null;
+    } finally {
+      postProcess();
+    }
+  }
+
 }

Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java?rev=1584857&r1=1584856&r2=1584857&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
(original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
Fri Apr  4 18:18:43 2014
@@ -118,6 +118,7 @@ import org.apache.hadoop.hbase.io.hfile.
 import org.apache.hadoop.hbase.io.hfile.PreloadThreadPool;
 import org.apache.hadoop.hbase.io.hfile.histogram.HFileHistogram;
 import org.apache.hadoop.hbase.io.hfile.histogram.HFileHistogram.Bucket;
+import org.apache.hadoop.hbase.io.hfile.histogram.HistogramUtils;
 import org.apache.hadoop.hbase.ipc.HBaseRPC;
 import org.apache.hadoop.hbase.ipc.HBaseRPCErrorHandler;
 import org.apache.hadoop.hbase.ipc.HBaseRPCOptions;
@@ -4061,6 +4062,21 @@ public class HRegionServer implements HR
         hist.getUniformBuckets(), region.getStartKey(), region.getEndKey());
   }
 
+  @Override
+  public List<List<Bucket>> getHistograms(List<byte[]> regionNames)
+      throws IOException {
+    List<List<Bucket>> ret = new ArrayList<>();
+    for (HRegion oregion : this.onlineRegions.values()) {
+      checkOpen();
+      HRegion region = getRegion(oregion.getRegionName());
+      HFileHistogram hist = region.getHistogram();
+      if (hist == null) return null;
+      ret.add(HRegionUtilities.adjustHistogramBoundariesToRegionBoundaries(
+        hist.getUniformBuckets(), region.getStartKey(), region.getEndKey()));
+    }
+    return ret;
+  }
+
   /**
    *
    * Returns null if no data is available.

Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/ThriftHRegionServer.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/ThriftHRegionServer.java?rev=1584857&r1=1584856&r2=1584857&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/ThriftHRegionServer.java
(original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/ThriftHRegionServer.java
Fri Apr  4 18:18:43 2014
@@ -59,6 +59,8 @@ import org.apache.hadoop.io.LongWritable
 import org.apache.hadoop.io.MapWritable;
 import org.apache.hadoop.io.Writable;
 
+import com.facebook.swift.service.ThriftException;
+import com.facebook.swift.service.ThriftMethod;
 import com.google.common.util.concurrent.Futures;
 import com.google.common.util.concurrent.ListenableFuture;
 
@@ -641,4 +643,11 @@ public class ThriftHRegionServer impleme
     return endpointServer.callEndpoint(epName, methodName, regionName,
         startRow, stopRow);
   }
+
+  @Override
+  @ThriftMethod(value = "getHistograms", exception = @ThriftException(type = ThriftHBaseException.class,
id = 1))
+  public List<List<Bucket>> getHistograms(List<byte[]> regionNames)
+      throws ThriftHBaseException {
+    return this.getHistograms(regionNames);
+  }
 }

Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestHFileHistogramE2E.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestHFileHistogramE2E.java?rev=1584857&r1=1584856&r2=1584857&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestHFileHistogramE2E.java
(original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestHFileHistogramE2E.java
Fri Apr  4 18:18:43 2014
@@ -205,5 +205,24 @@ public class TestHFileHistogramE2E {
       prevBucket = null;
       regionIndex++;
     }
+    int tries = 5;
+    while (tries-- >= 0) {
+      List<List<Bucket>> batchBuckets = null;
+      try {
+        batchBuckets =
+            table.batchgetHistogramsForAllRegions();
+      } catch (IOException e) {
+        continue;
+      }
+      compareHistograms(buckets, batchBuckets);
+    }
+  }
+
+  private void compareHistograms(List<List<Bucket>> buckets,
+      List<List<Bucket>> batchBuckets) {
+    assertTrue(buckets.size() == batchBuckets.size());
+    for (int i = 0; i < buckets.size(); i++) {
+      assertTrue(compareBuckets(buckets.get(i), batchBuckets.get(i)));
+    }
   }
 }



Mime
View raw message