hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From st...@apache.org
Subject hbase git commit: HBASE-14926 Hung ThriftServer; no timeout on read from client; if client crashes, worker thread gets stuck reading
Date Fri, 04 Dec 2015 22:04:31 GMT
Repository: hbase
Updated Branches:
  refs/heads/branch-1.2 f57bb78db -> 8954091df


HBASE-14926 Hung ThriftServer; no timeout on read from client; if client crashes, worker thread
gets stuck reading


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/8954091d
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/8954091d
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/8954091d

Branch: refs/heads/branch-1.2
Commit: 8954091dfba7950199bcab772e318028e6b66f0f
Parents: f57bb78
Author: stack <stack@apache.org>
Authored: Fri Dec 4 13:19:12 2015 -0800
Committer: stack <stack@apache.org>
Committed: Fri Dec 4 14:04:16 2015 -0800

----------------------------------------------------------------------
 hbase-examples/README.txt                       | 19 ++++----
 .../hadoop/hbase/thrift/ThriftServer.java       | 14 ++++--
 .../hadoop/hbase/thrift/ThriftServerRunner.java | 50 +++++++++++++-------
 .../hadoop/hbase/thrift2/ThriftServer.java      | 46 ++++++++++++++++--
 4 files changed, 97 insertions(+), 32 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/8954091d/hbase-examples/README.txt
----------------------------------------------------------------------
diff --git a/hbase-examples/README.txt b/hbase-examples/README.txt
index 700e41f..6578bb4 100644
--- a/hbase-examples/README.txt
+++ b/hbase-examples/README.txt
@@ -14,21 +14,24 @@ Example code.
     to be able to compile/run the examples without Thrift installed.
     If desired, the code can be re-generated as follows:
     thrift --gen cpp --gen java --gen rb --gen py --gen php --gen perl \
-        ${HBASE_ROOT}/hbase-server/src/main/resources/org/apache/hadoop/hbase/thrift/Hbase.thrift
-    and re-placed at the corresponding paths.
+        ${HBASE_ROOT}/hbase-thrift/src/main/resources/org/apache/hadoop/hbase/thrift/Hbase.thrift
+    and re-placed at the corresponding paths. You should not have to do this generally.
 
-    Before you run any Thrift examples, find a running HBase Thrift server.
-    If you start one locally (bin/hbase thrift start), the default port is 9090.
+    Before you run any Thrift examples, find a running HBase Thrift server (and a running
+    hbase cluster for this server to talk to -- at a minimum start a standalone instance
+    by doing ./bin/start-hbase.sh). If you start one locally (bin/hbase thrift start),
+    the default port is 9090 (a webserver with basic stats defaults showing on port 9095).
 
     * Java: org.apache.hadoop.hbase.thrift.DemoClient (jar under lib/).
-      1. Set up the classpath with all the necessary jars, for example:
-        for f in `find . -name "libthrift-*.jar" -or -name "slf4j-*.jar" -or -name "log4j-*.jar"`;
do
-          HBASE_EXAMPLE_CLASSPATH=${HBASE_EXAMPLE_CLASSPATH}:$f;
-        done
+      1. Make sure your client has all required jars on the CLASSPATH when it starts. If
lazy,
+      just add all jars as follows: {HBASE_EXAMPLE_CLASSPATH=`./bin/hbase classpath`}
       2. If HBase server is not secure, or authentication is not enabled for the Thrift server,
execute:
       {java -cp hbase-examples-[VERSION].jar:${HBASE_EXAMPLE_CLASSPATH} org.apache.hadoop.hbase.thrift.DemoClient
<host> <port>}
       3. If HBase server is secure, and authentication is enabled for the Thrift server,
run kinit at first, then execute:
       {java -cp hbase-examples-[VERSION].jar:${HBASE_EXAMPLE_CLASSPATH} org.apache.hadoop.hbase.thrift.DemoClient
<host> <port> true}
+      4. Here is a lazy example that just pulls in all hbase dependency jars and that goes
against default location on localhost.
+      It should work with a standalone hbase instance started by doing ./bin/start-hbase.sh:
+      {java -cp ./hbase-examples/target/hbase-examples-2.0.0-SNAPSHOT.jar:`./bin/hbase classpath`
org.apache.hadoop.hbase.thrift.DemoClient localhost 9090}
 
     * Ruby: hbase-examples/src/main/ruby/DemoClient.rb
       1. Modify the import path in the file to point to {$THRIFT_HOME}/lib/rb/lib.

http://git-wip-us.apache.org/repos/asf/hbase/blob/8954091d/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServer.java
----------------------------------------------------------------------
diff --git a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServer.java b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServer.java
index 59c7e2d..560f788 100644
--- a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServer.java
+++ b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServer.java
@@ -64,6 +64,8 @@ public class ThriftServer {
 
   private InfoServer infoServer;
 
+  private static final String READ_TIMEOUT_OPTION = "readTimeout";
+
   //
   // Main program and support routines
   //
@@ -134,6 +136,11 @@ public class ThriftServer {
         "The amount of time in secods to keep a thread alive when idle in " +
         ImplType.THREAD_POOL.simpleClassName());
 
+    options.addOption("t", READ_TIMEOUT_OPTION, true,
+        "Amount of time in milliseconds before a server thread will timeout " +
+        "waiting for client to send data on a connected socket. Currently, " +
+        "only applies to TBoundedThreadPoolServer");
+
     options.addOptionGroup(ImplType.createOptionGroup());
 
     CommandLineParser parser = new PosixParser();
@@ -185,7 +192,9 @@ public class ThriftServer {
         conf, TBoundedThreadPoolServer.MAX_QUEUED_REQUESTS_CONF_KEY);
     optionToConf(cmd, KEEP_ALIVE_SEC_OPTION,
         conf, TBoundedThreadPoolServer.THREAD_KEEP_ALIVE_TIME_SEC_CONF_KEY);
-
+    optionToConf(cmd, READ_TIMEOUT_OPTION, conf,
+        ThriftServerRunner.THRIFT_SERVER_SOCKET_READ_TIMEOUT_KEY);
+    
     // Set general thrift server options
     boolean compact = cmd.hasOption(COMPACT_OPTION) ||
       conf.getBoolean(ThriftServerRunner.COMPACT_CONF_KEY, false);
@@ -194,8 +203,7 @@ public class ThriftServer {
       conf.getBoolean(ThriftServerRunner.FRAMED_CONF_KEY, false);
     conf.setBoolean(ThriftServerRunner.FRAMED_CONF_KEY, framed);
     if (cmd.hasOption(BIND_OPTION)) {
-      conf.set(
-          ThriftServerRunner.BIND_CONF_KEY, cmd.getOptionValue(BIND_OPTION));
+      conf.set(ThriftServerRunner.BIND_CONF_KEY, cmd.getOptionValue(BIND_OPTION));
     }
 
     ImplType.setServerImpl(cmd, conf);

http://git-wip-us.apache.org/repos/asf/hbase/blob/8954091d/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServerRunner.java
----------------------------------------------------------------------
diff --git a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServerRunner.java
b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServerRunner.java
index 048c1f9..9688131 100644
--- a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServerRunner.java
+++ b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServerRunner.java
@@ -160,6 +160,15 @@ public class ThriftServerRunner implements Runnable {
   static final String THRIFT_SSL_KEYSTORE_PASSWORD = "hbase.thrift.ssl.keystore.password";
   static final String THRIFT_SSL_KEYSTORE_KEYPASSWORD = "hbase.thrift.ssl.keystore.keypassword";
 
+  /**
+   * Amount of time in milliseconds before a server thread will timeout
+   * waiting for client to send data on a connected socket. Currently,
+   * applies only to TBoundedThreadPoolServer
+   */
+  public static final String THRIFT_SERVER_SOCKET_READ_TIMEOUT_KEY =
+    "hbase.thrift.server.socket.read.timeout";
+  public static final int THRIFT_SERVER_SOCKET_READ_TIMEOUT_DEFAULT = 60000;
+
 
   /**
    * Thrift quality of protection configuration key. Valid values can be:
@@ -171,6 +180,7 @@ public class ThriftServerRunner implements Runnable {
    * The thrift server and the HBase cluster must run in secure mode.
    */
   static final String THRIFT_QOP_KEY = "hbase.thrift.security.qop";
+  static final String BACKLOG_CONF_KEY = "hbase.regionserver.thrift.backlog";
 
   private static final String DEFAULT_BIND_ADDR = "0.0.0.0";
   public static final int DEFAULT_LISTEN_PORT = 9090;
@@ -516,9 +526,11 @@ public class ThriftServerRunner implements Runnable {
           "-" + BIND_CONF_KEY + " not supported with " + implType);
     }
 
+    // Thrift's implementation uses '0' as a placeholder for 'use the default.'
+    int backlog = conf.getInt(BACKLOG_CONF_KEY, 0);
+
     if (implType == ImplType.HS_HA || implType == ImplType.NONBLOCKING ||
         implType == ImplType.THREADED_SELECTOR) {
-
       InetAddress listenAddress = getBindAddress(conf);
       TNonblockingServerTransport serverTransport = new TNonblockingServerSocket(
           new InetSocketAddress(listenAddress, listenPort));
@@ -559,9 +571,13 @@ public class ThriftServerRunner implements Runnable {
     } else if (implType == ImplType.THREAD_POOL) {
       // Thread pool server. Get the IP address to bind to.
       InetAddress listenAddress = getBindAddress(conf);
-
+      int readTimeout = conf.getInt(THRIFT_SERVER_SOCKET_READ_TIMEOUT_KEY,
+          THRIFT_SERVER_SOCKET_READ_TIMEOUT_DEFAULT);
       TServerTransport serverTransport = new TServerSocket(
-          new InetSocketAddress(listenAddress, listenPort));
+          new TServerSocket.ServerSocketTransportArgs().
+              bindAddr(new InetSocketAddress(listenAddress, listenPort)).
+              backlog(backlog).
+              clientTimeout(readTimeout));
 
       TBoundedThreadPoolServer.Args serverArgs =
           new TBoundedThreadPoolServer.Args(serverTransport, conf);
@@ -570,7 +586,7 @@ public class ThriftServerRunner implements Runnable {
                 .protocolFactory(protocolFactory);
       LOG.info("starting " + ImplType.THREAD_POOL.simpleClassName() + " on "
           + listenAddress + ":" + Integer.toString(listenPort)
-          + "; " + serverArgs);
+          + " with readTimeout " + readTimeout + "ms; " + serverArgs);
       TBoundedThreadPoolServer tserver =
           new TBoundedThreadPoolServer(serverArgs, metrics);
       this.tserver = tserver;
@@ -911,7 +927,7 @@ public class ThriftServerRunner implements Runnable {
      */
     public List<TCell> getVer(ByteBuffer tableName, ByteBuffer row, byte[] family,
         byte[] qualifier, int numVersions, Map<ByteBuffer, ByteBuffer> attributes)
throws IOError {
-      
+
       Table table = null;
       try {
         table = getTable(tableName);
@@ -957,7 +973,7 @@ public class ThriftServerRunner implements Runnable {
     protected List<TCell> getVerTs(ByteBuffer tableName, ByteBuffer row, byte[] family,
         byte[] qualifier, long timestamp, int numVersions, Map<ByteBuffer, ByteBuffer>
attributes)
         throws IOError {
-      
+
       Table table = null;
       try {
         table = getTable(tableName);
@@ -1009,7 +1025,7 @@ public class ThriftServerRunner implements Runnable {
     public List<TRowResult> getRowWithColumnsTs(
         ByteBuffer tableName, ByteBuffer row, List<ByteBuffer> columns,
         long timestamp, Map<ByteBuffer, ByteBuffer> attributes) throws IOError {
-      
+
       Table table = null;
       try {
         table = getTable(tableName);
@@ -1075,7 +1091,7 @@ public class ThriftServerRunner implements Runnable {
                                                  List<ByteBuffer> rows,
         List<ByteBuffer> columns, long timestamp,
         Map<ByteBuffer, ByteBuffer> attributes) throws IOError {
-      
+
       Table table= null;
       try {
         List<Get> gets = new ArrayList<Get>(rows.size());
@@ -1424,7 +1440,7 @@ public class ThriftServerRunner implements Runnable {
     public int scannerOpenWithScan(ByteBuffer tableName, TScan tScan,
         Map<ByteBuffer, ByteBuffer> attributes)
         throws IOError {
-      
+
       Table table = null;
       try {
         table = getTable(tableName);
@@ -1476,7 +1492,7 @@ public class ThriftServerRunner implements Runnable {
     public int scannerOpen(ByteBuffer tableName, ByteBuffer startRow,
         List<ByteBuffer> columns,
         Map<ByteBuffer, ByteBuffer> attributes) throws IOError {
-      
+
       Table table = null;
       try {
         table = getTable(tableName);
@@ -1506,7 +1522,7 @@ public class ThriftServerRunner implements Runnable {
         ByteBuffer stopRow, List<ByteBuffer> columns,
         Map<ByteBuffer, ByteBuffer> attributes)
         throws IOError, TException {
-      
+
       Table table = null;
       try {
         table = getTable(tableName);
@@ -1537,7 +1553,7 @@ public class ThriftServerRunner implements Runnable {
                                      List<ByteBuffer> columns,
         Map<ByteBuffer, ByteBuffer> attributes)
         throws IOError, TException {
-      
+
       Table table = null;
       try {
         table = getTable(tableName);
@@ -1569,7 +1585,7 @@ public class ThriftServerRunner implements Runnable {
     public int scannerOpenTs(ByteBuffer tableName, ByteBuffer startRow,
         List<ByteBuffer> columns, long timestamp,
         Map<ByteBuffer, ByteBuffer> attributes) throws IOError, TException {
-      
+
       Table table = null;
       try {
         table = getTable(tableName);
@@ -1600,7 +1616,7 @@ public class ThriftServerRunner implements Runnable {
         ByteBuffer stopRow, List<ByteBuffer> columns, long timestamp,
         Map<ByteBuffer, ByteBuffer> attributes)
         throws IOError, TException {
-      
+
       Table table = null;
       try {
         table = getTable(tableName);
@@ -1630,7 +1646,7 @@ public class ThriftServerRunner implements Runnable {
     @Override
     public Map<ByteBuffer, ColumnDescriptor> getColumnDescriptors(
         ByteBuffer tableName) throws IOError, TException {
-      
+
       Table table = null;
       try {
         TreeMap<ByteBuffer, ColumnDescriptor> columns =
@@ -1715,13 +1731,13 @@ public class ThriftServerRunner implements Runnable {
         throw new IOError(Throwables.getStackTraceAsString(e));
       }
     }
-    
+
     private Result getRowOrBefore(byte[] tableName, byte[] row, byte[] family) throws IOException
{
       Scan scan = new Scan(row);
       scan.setReversed(true);
       scan.addFamily(family);
       scan.setStartRow(row);
-      Table table = getTable(tableName);      
+      Table table = getTable(tableName);
       try (ResultScanner scanner = table.getScanner(scan)) {
         return scanner.next();
       } finally{

http://git-wip-us.apache.org/repos/asf/hbase/blob/8954091d/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftServer.java
----------------------------------------------------------------------
diff --git a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftServer.java
b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftServer.java
index 480e7c2..429475e 100644
--- a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftServer.java
+++ b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftServer.java
@@ -108,6 +108,18 @@ public class ThriftServer {
 
   public static final int DEFAULT_LISTEN_PORT = 9090;
 
+  private static final String READ_TIMEOUT_OPTION = "readTimeout";
+
+  static final String BACKLOG_CONF_KEY = "hbase.regionserver.thrift.backlog";
+
+  /**
+   * Amount of time in milliseconds before a server thread will timeout
+   * waiting for client to send data on a connected socket. Currently,
+   * applies only to TBoundedThreadPoolServer
+   */
+  public static final String THRIFT_SERVER_SOCKET_READ_TIMEOUT_KEY =
+    "hbase.thrift.server.socket.read.timeout";
+  public static final int THRIFT_SERVER_SOCKET_READ_TIMEOUT_DEFAULT = 60000;
 
   public ThriftServer() {
   }
@@ -131,7 +143,10 @@ public class ThriftServer {
     options.addOption("w", "workers", true, "How many worker threads to use.");
     options.addOption("h", "help", false, "Print help information");
     options.addOption(null, "infoport", true, "Port for web UI");
-
+    options.addOption("t", READ_TIMEOUT_OPTION, true,
+      "Amount of time in milliseconds before a server thread will timeout " +
+      "waiting for client to send data on a connected socket. Currently, " +
+      "only applies to TBoundedThreadPoolServer");
     OptionGroup servers = new OptionGroup();
     servers.addOption(
         new Option("nonblocking", false, "Use the TNonblockingServer. This implies the framed
transport."));
@@ -272,9 +287,14 @@ public class ThriftServer {
                                               TProcessor processor,
                                               TTransportFactory transportFactory,
                                               int workerThreads,
-                                              InetSocketAddress inetSocketAddress)
+                                              InetSocketAddress inetSocketAddress,
+                                              int backlog,
+                                              int clientTimeout)
       throws TTransportException {
-    TServerTransport serverTransport = new TServerSocket(inetSocketAddress);
+    TServerTransport serverTransport = new TServerSocket(
+                                           new TServerSocket.ServerSocketTransportArgs().
+                                               bindAddr(inetSocketAddress).backlog(backlog).
+                                               clientTimeout(clientTimeout));
     log.info("starting HBase ThreadPool Thrift server on " + inetSocketAddress.toString());
     TThreadPoolServer.Args serverArgs = new TThreadPoolServer.Args(serverTransport);
     serverArgs.processor(processor);
@@ -336,6 +356,19 @@ public class ThriftServer {
       bindAddress = conf.get("hbase.thrift.info.bindAddress");
     }
 
+    // Get read timeout
+    int readTimeout = THRIFT_SERVER_SOCKET_READ_TIMEOUT_DEFAULT;
+    if (cmd.hasOption(READ_TIMEOUT_OPTION)) {
+      try {
+        readTimeout = Integer.parseInt(cmd.getOptionValue(READ_TIMEOUT_OPTION));
+      } catch (NumberFormatException e) {
+        throw new RuntimeException("Could not parse the value provided for the timeout option",
e);
+      }
+    } else {
+      readTimeout = conf.getInt(THRIFT_SERVER_SOCKET_READ_TIMEOUT_KEY,
+        THRIFT_SERVER_SOCKET_READ_TIMEOUT_DEFAULT);
+    }
+
     // Get port to bind to
     int listenPort = 0;
     try {
@@ -348,6 +381,9 @@ public class ThriftServer {
       throw new RuntimeException("Could not parse the value provided for the port option",
e);
     }
 
+    // Thrift's implementation uses '0' as a placeholder for 'use the default.'
+    int backlog = conf.getInt(BACKLOG_CONF_KEY, 0);
+
     // Local hostname and user name,
     // used only if QOP is configured.
     String host = null;
@@ -473,7 +509,9 @@ public class ThriftServer {
           processor,
           transportFactory,
           workerThreads,
-          inetSocketAddress);
+          inetSocketAddress,
+          backlog,
+          readTimeout);
     }
 
     final TServer tserver = server;


Mime
View raw message