Return-Path: X-Original-To: apmail-hadoop-common-commits-archive@www.apache.org Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 18BA29EDB for ; Fri, 24 Feb 2012 14:50:03 +0000 (UTC) Received: (qmail 4454 invoked by uid 500); 24 Feb 2012 14:50:02 -0000 Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org Received: (qmail 4396 invoked by uid 500); 24 Feb 2012 14:50:02 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-dev@hadoop.apache.org Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 4384 invoked by uid 99); 24 Feb 2012 14:50:02 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 24 Feb 2012 14:50:02 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 24 Feb 2012 14:50:00 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id B9A8A23889E2; Fri, 24 Feb 2012 14:49:40 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1293279 - in /hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common: ./ src/main/docs/ src/main/java/ src/main/java/org/apache/hadoop/ipc/ src/main/java/org/apache/hadoop/net/ src/test/core/ src/test/java/org/apache/hadoop/ipc/ Date: Fri, 24 Feb 2012 14:49:40 -0000 To: common-commits@hadoop.apache.org From: atm@apache.org X-Mailer: svnmailer-1.0.8-patched Message-Id: <20120224144940.B9A8A23889E2@eris.apache.org> Author: atm Date: Fri Feb 24 14:49:38 2012 New Revision: 1293279 URL: http://svn.apache.org/viewvc?rev=1293279&view=rev Log: Merge trunk into HA branch. Modified: hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/CHANGES.txt (contents, props changed) hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/docs/ (props changed) hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/ (props changed) hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/test/core/ (props changed) hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestProtoBufRpc.java Modified: hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1293279&r1=1293278&r2=1293279&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/CHANGES.txt Fri Feb 24 14:49:38 2012 @@ -89,6 +89,12 @@ Trunk (unreleased changes) HADOOP-8084. Updates ProtoBufRpc engine to not do an unnecessary copy for RPC request/response. (ddas) + HADOOP-8085. Add RPC metrics to ProtobufRpcEngine. (Hari Mankude via + suresh) + + HADOOP-8108. Move method getHostPortString() from NameNode to NetUtils. + (Brandon Li via jitendra) + BUG FIXES HADOOP-8018. Hudson auto test for HDFS has started throwing javadoc Propchange: hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/CHANGES.txt ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Fri Feb 24 14:49:38 2012 @@ -1,6 +1,6 @@ /hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt:1243206,1291081 /hadoop/common/branches/yahoo-merge/CHANGES.txt:1079157,1079163-1079164,1079167 -/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt:1161333-1292826 +/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt:1161333-1293274 /hadoop/core/branches/branch-0.18/CHANGES.txt:727226 /hadoop/core/branches/branch-0.19/CHANGES.txt:713112 /hadoop/core/trunk/CHANGES.txt:776175-785643,785929-786278 Propchange: hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/docs/ ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Fri Feb 24 14:49:38 2012 @@ -1,2 +1,2 @@ -/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/docs:1152502-1292826 +/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/docs:1152502-1293274 /hadoop/core/branches/branch-0.19/src/docs:713112 Propchange: hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/ ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Fri Feb 24 14:49:38 2012 @@ -1,3 +1,3 @@ -/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java:1152502-1292826 +/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java:1152502-1293274 /hadoop/core/branches/branch-0.19/core/src/java:713112 /hadoop/core/trunk/src/core:776175-785643,785929-786278 Modified: hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java?rev=1293279&r1=1293278&r2=1293279&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java Fri Feb 24 14:49:38 2012 @@ -379,6 +379,24 @@ public class ProtobufRpcEngine implement * Protobuf invoker for {@link RpcInvoker} */ static class ProtoBufRpcInvoker implements RpcInvoker { + private static ProtoClassProtoImpl getProtocolImpl(RPC.Server server, + String protoName, long version) throws IOException { + ProtoNameVer pv = new ProtoNameVer(protoName, version); + ProtoClassProtoImpl impl = + server.getProtocolImplMap(RpcKind.RPC_PROTOCOL_BUFFER).get(pv); + if (impl == null) { // no match for Protocol AND Version + VerProtocolImpl highest = + server.getHighestSupportedProtocol(RpcKind.RPC_PROTOCOL_BUFFER, + protoName); + if (highest == null) { + throw new IOException("Unknown protocol: " + protoName); + } + // protocol supported but not the version that client wants + throw new RPC.VersionMismatch(protoName, version, + highest.version); + } + return impl; + } @Override /** @@ -409,21 +427,8 @@ public class ProtobufRpcEngine implement if (server.verbose) LOG.info("Call: protocol=" + protocol + ", method=" + methodName); - ProtoNameVer pv = new ProtoNameVer(protoName, clientVersion); - ProtoClassProtoImpl protocolImpl = - server.getProtocolImplMap(RpcKind.RPC_PROTOCOL_BUFFER).get(pv); - if (protocolImpl == null) { // no match for Protocol AND Version - VerProtocolImpl highest = - server.getHighestSupportedProtocol(RpcKind.RPC_PROTOCOL_BUFFER, - protoName); - if (highest == null) { - throw new IOException("Unknown protocol: " + protoName); - } - // protocol supported but not the version that client wants - throw new RPC.VersionMismatch(protoName, clientVersion, - highest.version); - } - + ProtoClassProtoImpl protocolImpl = getProtocolImpl(server, protoName, + clientVersion); BlockingService service = (BlockingService) protocolImpl.protocolImpl; MethodDescriptor methodDescriptor = service.getDescriptorForType() .findMethodByName(methodName); @@ -438,7 +443,19 @@ public class ProtobufRpcEngine implement .mergeFrom(rpcRequest.getRequest()).build(); Message result; try { + long startTime = System.currentTimeMillis(); + server.rpcDetailedMetrics.init(protocolImpl.protocolClass); result = service.callBlockingMethod(methodDescriptor, null, param); + int processingTime = (int) (System.currentTimeMillis() - startTime); + int qTime = (int) (startTime - receiveTime); + if (LOG.isDebugEnabled()) { + LOG.info("Served: " + methodName + " queueTime= " + qTime + + " procesingTime= " + processingTime); + } + server.rpcMetrics.addRpcQueueTime(qTime); + server.rpcMetrics.addRpcProcessingTime(processingTime); + server.rpcDetailedMetrics.addProcessingTime(methodName, + processingTime); } catch (ServiceException e) { Throwable cause = e.getCause(); return handleException(cause != null ? cause : e); Modified: hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java?rev=1293279&r1=1293278&r2=1293279&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java Fri Feb 24 14:49:38 2012 @@ -317,10 +317,15 @@ public abstract class Server { * Returns a handle to the rpcMetrics (required in tests) * @return rpc metrics */ + @VisibleForTesting public RpcMetrics getRpcMetrics() { return rpcMetrics; } + @VisibleForTesting + public RpcDetailedMetrics getRpcDetailedMetrics() { + return rpcDetailedMetrics; + } @VisibleForTesting Iterable getHandlers() { Modified: hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java?rev=1293279&r1=1293278&r2=1293279&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java Fri Feb 24 14:49:38 2012 @@ -607,6 +607,13 @@ public class NetUtils { } /** + * Compose a "host:port" string from the address. + */ + public static String getHostPortString(InetSocketAddress addr) { + return addr.getHostName() + ":" + addr.getPort(); + } + + /** * Checks if {@code host} is a local host name and return {@link InetAddress} * corresponding to that address. * Propchange: hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/test/core/ ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Fri Feb 24 14:49:38 2012 @@ -1,3 +1,3 @@ -/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/core:1152502-1292826 +/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/core:1152502-1293274 /hadoop/core/branches/branch-0.19/core/src/test/core:713112 /hadoop/core/trunk/src/test/core:776175-785643,785929-786278 Modified: hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestProtoBufRpc.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestProtoBufRpc.java?rev=1293279&r1=1293278&r2=1293279&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestProtoBufRpc.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestProtoBufRpc.java Fri Feb 24 14:49:38 2012 @@ -17,6 +17,9 @@ */ package org.apache.hadoop.ipc; +import static org.apache.hadoop.test.MetricsAsserts.getMetrics; +import static org.apache.hadoop.test.MetricsAsserts.assertCounterGt; + import java.io.IOException; import java.net.InetSocketAddress; @@ -28,6 +31,7 @@ import org.apache.hadoop.ipc.protobuf.Te import org.apache.hadoop.ipc.protobuf.TestProtos.EmptyResponseProto; import org.apache.hadoop.ipc.protobuf.TestRpcServiceProtos.TestProtobufRpcProto; import org.apache.hadoop.ipc.protobuf.TestRpcServiceProtos.TestProtobufRpc2Proto; +import org.apache.hadoop.metrics2.MetricsRecordBuilder; import org.apache.hadoop.net.NetUtils; import org.junit.Assert; import org.junit.Test; @@ -187,5 +191,14 @@ public class TestProtoBufRpc { .setMessage("hello").build(); EchoResponseProto echoResponse = client.echo2(null, echoRequest); Assert.assertEquals(echoResponse.getMessage(), "hello"); + + // Ensure RPC metrics are updated + MetricsRecordBuilder rpcMetrics = getMetrics(server.getRpcMetrics().name()); + assertCounterGt("RpcQueueTimeNumOps", 0L, rpcMetrics); + assertCounterGt("RpcProcessingTimeNumOps", 0L, rpcMetrics); + + MetricsRecordBuilder rpcDetailedMetrics = + getMetrics(server.getRpcDetailedMetrics().name()); + assertCounterGt("Echo2NumOps", 0L, rpcDetailedMetrics); } } \ No newline at end of file