hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From szets...@apache.org
Subject svn commit: r1293344 - in /hadoop/common/branches/branch-0.23-PB/hadoop-common-project/hadoop-common: ./ src/main/java/ src/main/java/org/apache/hadoop/ipc/ src/test/java/org/apache/hadoop/ipc/
Date Fri, 24 Feb 2012 17:14:13 GMT
Author: szetszwo
Date: Fri Feb 24 17:14:12 2012
New Revision: 1293344

URL: http://svn.apache.org/viewvc?rev=1293344&view=rev
Log:
svn merge -c 1293071 from trunk for HADOOP-8085.

Modified:
    hadoop/common/branches/branch-0.23-PB/hadoop-common-project/hadoop-common/   (props changed)
    hadoop/common/branches/branch-0.23-PB/hadoop-common-project/hadoop-common/CHANGES.txt
  (contents, props changed)
    hadoop/common/branches/branch-0.23-PB/hadoop-common-project/hadoop-common/src/main/java/
  (props changed)
    hadoop/common/branches/branch-0.23-PB/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java
    hadoop/common/branches/branch-0.23-PB/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
    hadoop/common/branches/branch-0.23-PB/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestProtoBufRpc.java

Propchange: hadoop/common/branches/branch-0.23-PB/hadoop-common-project/hadoop-common/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Feb 24 17:14:12 2012
@@ -1,2 +1,2 @@
 /hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common:1227776-1293033
-/hadoop/common/trunk/hadoop-common-project/hadoop-common:1161777,1161781,1162008,1162188,1162421,1162491,1162499,1162613,1162928,1162954,1162979,1163050,1163069,1163456,1163465,1163490,1163768,1163852,1163858,1164255,1164301,1164339,1164771,1166009,1166402,1167318,1167383,1170379,1170459,1171221,1171297,1172916,1173402,1176550,1176986,1177002,1177035,1177399,1177487,1177531,1177859,1177864,1178639,1179512,1179869,1182189,1182205,1182214,1182641,1183132,1189357,1189613,1189932,1189982,1190109,1190611,1195575,1195760,1196113,1196129,1197885,1204114,1204117,1204122,1204124,1204129,1204131,1204177,1204363,1204370,1204376,1204388,1205260,1205697,1206786,1206830,1207694,1208153,1208313,1209246,1210208,1210319-1210320,1212004,1212021,1212062,1212073,1212084,1212615,1213537,1213586,1213592-1213593,1213598,1213619,1213954,1214046,1214114,1214499,1215358,1220510,1221348,1225114,1225192,1225456,1225489,1225591,1226211,1226239,1226350,1227091,1227165,1227423,1227861,1227964,1229347,1230
 398,1231569,1231572,1231627,1231640,1231732,1233605,1234555,1235135,1235137,1235956,1236444,1236456,1239386,1239752,1240897,1240928,1242485,1243065,1243104,1244185,1244766,1245751,1245762,1291602
+/hadoop/common/trunk/hadoop-common-project/hadoop-common:1161777,1161781,1162008,1162188,1162421,1162491,1162499,1162613,1162928,1162954,1162979,1163050,1163069,1163456,1163465,1163490,1163768,1163852,1163858,1164255,1164301,1164339,1164771,1166009,1166402,1167318,1167383,1170379,1170459,1171221,1171297,1172916,1173402,1176550,1176986,1177002,1177035,1177399,1177487,1177531,1177859,1177864,1178639,1179512,1179869,1182189,1182205,1182214,1182641,1183132,1189357,1189613,1189932,1189982,1190109,1190611,1195575,1195760,1196113,1196129,1197885,1204114,1204117,1204122,1204124,1204129,1204131,1204177,1204363,1204370,1204376,1204388,1205260,1205697,1206786,1206830,1207694,1208153,1208313,1209246,1210208,1210319-1210320,1212004,1212021,1212062,1212073,1212084,1212615,1213537,1213586,1213592-1213593,1213598,1213619,1213954,1214046,1214114,1214499,1215358,1220510,1221348,1225114,1225192,1225456,1225489,1225591,1226211,1226239,1226350,1227091,1227165,1227423,1227861,1227964,1229347,1230
 398,1231569,1231572,1231627,1231640,1231732,1233605,1234555,1235135,1235137,1235956,1236444,1236456,1239386,1239752,1240897,1240928,1242485,1243065,1243104,1244185,1244766,1245751,1245762,1291602,1293071

Modified: hadoop/common/branches/branch-0.23-PB/hadoop-common-project/hadoop-common/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1293344&r1=1293343&r2=1293344&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23-PB/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++ hadoop/common/branches/branch-0.23-PB/hadoop-common-project/hadoop-common/CHANGES.txt
Fri Feb 24 17:14:12 2012
@@ -50,6 +50,9 @@ Release 0.23-PB - Unreleased
     HADOOP-8084. Updates ProtoBufRpc engine to not do an unnecessary copy 
     for RPC request/response. (ddas)
 
+    HADOOP-8085. Add RPC metrics to ProtobufRpcEngine. (Hari Mankude via
+    suresh)
+
   BUG FIXES
 
     HADOOP-7695. RPC.stopProxy can throw unintended exception while logging

Propchange: hadoop/common/branches/branch-0.23-PB/hadoop-common-project/hadoop-common/CHANGES.txt
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Feb 24 17:14:12 2012
@@ -1,6 +1,6 @@
 /hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt:1227776-1293033
 /hadoop/common/branches/yahoo-merge/CHANGES.txt:1079157,1079163-1079164,1079167
-/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt:1161777,1161781,1162008,1162188,1162421,1162491,1162499,1162613,1162928,1162954,1162979,1163050,1163069,1163456,1163465,1163490,1163768,1163852,1163858,1164255,1164301,1164771,1166009,1166402,1167318,1167383,1169986,1170046,1170379,1170459,1171221,1171297,1171894,1171909,1172186,1172916,1173402,1176550,1176986,1177002,1177035,1177399,1177487,1177531,1177859,1177864,1178639,1179512,1179869,1182189,1182205,1182214,1182641,1183132,1189357,1189613,1189932,1189982,1190109,1190611,1195575,1195760,1196113,1196129,1197885,1204114,1204117,1204122,1204124,1204129,1204131,1204177,1204363,1204376,1204388,1205260,1206830,1207694,1208153,1208313,1209246,1210208,1210319-1210320,1212004,1212062,1212073,1212084,1212615,1213537,1213586,1213592-1213593,1213598,1213619,1213954,1214046,1214114,1214499,1215358,1220510,1221348,1225114,1225192,1225456,1225489,1225591,1226211,1226239,1226350-1226351,1227091,1227165,1227423,1227861,
 1227964,1229347,1230398,1231569,1231572,1231627,1231640,1231732,1233605,1234555,1235135,1235137,1235956,1236444,1236456,1239386,1239752,1240897,1240928,1242485,1243065,1243104,1244185,1244766,1245751,1245762,1291602
+/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt:1161777,1161781,1162008,1162188,1162421,1162491,1162499,1162613,1162928,1162954,1162979,1163050,1163069,1163456,1163465,1163490,1163768,1163852,1163858,1164255,1164301,1164771,1166009,1166402,1167318,1167383,1169986,1170046,1170379,1170459,1171221,1171297,1171894,1171909,1172186,1172916,1173402,1176550,1176986,1177002,1177035,1177399,1177487,1177531,1177859,1177864,1178639,1179512,1179869,1182189,1182205,1182214,1182641,1183132,1189357,1189613,1189932,1189982,1190109,1190611,1195575,1195760,1196113,1196129,1197885,1204114,1204117,1204122,1204124,1204129,1204131,1204177,1204363,1204376,1204388,1205260,1206830,1207694,1208153,1208313,1209246,1210208,1210319-1210320,1212004,1212062,1212073,1212084,1212615,1213537,1213586,1213592-1213593,1213598,1213619,1213954,1214046,1214114,1214499,1215358,1220510,1221348,1225114,1225192,1225456,1225489,1225591,1226211,1226239,1226350-1226351,1227091,1227165,1227423,1227861,
 1227964,1229347,1230398,1231569,1231572,1231627,1231640,1231732,1233605,1234555,1235135,1235137,1235956,1236444,1236456,1239386,1239752,1240897,1240928,1242485,1243065,1243104,1244185,1244766,1245751,1245762,1291602,1293071
 /hadoop/core/branches/branch-0.18/CHANGES.txt:727226
 /hadoop/core/branches/branch-0.19/CHANGES.txt:713112
 /hadoop/core/trunk/CHANGES.txt:776175-785643,785929-786278

Propchange: hadoop/common/branches/branch-0.23-PB/hadoop-common-project/hadoop-common/src/main/java/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Feb 24 17:14:12 2012
@@ -1,4 +1,4 @@
 /hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java:1227776-1293033
-/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java:1161777,1161781,1162008,1162188,1162421,1162491,1162499,1162613,1162928,1162954,1162979,1163050,1163069,1163456,1163465,1163490,1163768,1163852,1163858,1164255,1164301,1164339,1164771,1166402,1167318,1167383,1170379,1170459,1171221,1171297,1172916,1173402,1176550,1176986,1177002,1177035,1177399,1177487,1177531,1177859,1177864,1178639,1179512,1182189,1182205,1182214,1182641,1183132,1189613,1189932,1189982,1190109,1190611,1195575,1195760,1196113,1196129,1197885,1204114,1204117,1204122,1204124,1204129,1204131,1204177,1204363,1204370,1204376,1204388,1205260,1205697,1206786,1206830,1207694,1208153,1208313,1210208,1210319-1210320,1212004,1212021,1212062,1212073,1212084,1212615,1213537,1213586,1213592-1213593,1213619,1213954,1214046,1214114,1214499,1215358,1220510,1221348,1225114,1225192,1225456,1225489,1225591,1226211,1226239,1226350,1227091,1227165,1227423,1227861,1227964,1229347,1230398,1231569,1231572,123162
 7,1231640,1231732,1233605,1234555,1235135,1235137,1235956,1236444,1236456,1239752,1240897,1240928,1243065,1243104,1244185,1244766,1245751,1245762,1291602
+/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java:1161777,1161781,1162008,1162188,1162421,1162491,1162499,1162613,1162928,1162954,1162979,1163050,1163069,1163456,1163465,1163490,1163768,1163852,1163858,1164255,1164301,1164339,1164771,1166402,1167318,1167383,1170379,1170459,1171221,1171297,1172916,1173402,1176550,1176986,1177002,1177035,1177399,1177487,1177531,1177859,1177864,1178639,1179512,1182189,1182205,1182214,1182641,1183132,1189613,1189932,1189982,1190109,1190611,1195575,1195760,1196113,1196129,1197885,1204114,1204117,1204122,1204124,1204129,1204131,1204177,1204363,1204370,1204376,1204388,1205260,1205697,1206786,1206830,1207694,1208153,1208313,1210208,1210319-1210320,1212004,1212021,1212062,1212073,1212084,1212615,1213537,1213586,1213592-1213593,1213619,1213954,1214046,1214114,1214499,1215358,1220510,1221348,1225114,1225192,1225456,1225489,1225591,1226211,1226239,1226350,1227091,1227165,1227423,1227861,1227964,1229347,1230398,1231569,1231572,123162
 7,1231640,1231732,1233605,1234555,1235135,1235137,1235956,1236444,1236456,1239752,1240897,1240928,1243065,1243104,1244185,1244766,1245751,1245762,1291602,1293071
 /hadoop/core/branches/branch-0.19/core/src/java:713112
 /hadoop/core/trunk/src/core:776175-785643,785929-786278

Modified: hadoop/common/branches/branch-0.23-PB/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java?rev=1293344&r1=1293343&r2=1293344&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23-PB/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java
(original)
+++ hadoop/common/branches/branch-0.23-PB/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java
Fri Feb 24 17:14:12 2012
@@ -379,6 +379,24 @@ public class ProtobufRpcEngine implement
      * Protobuf invoker for {@link RpcInvoker}
      */
     static class ProtoBufRpcInvoker implements RpcInvoker {
+      private static ProtoClassProtoImpl getProtocolImpl(RPC.Server server,
+          String protoName, long version) throws IOException {
+        ProtoNameVer pv = new ProtoNameVer(protoName, version);
+        ProtoClassProtoImpl impl = 
+            server.getProtocolImplMap(RpcKind.RPC_PROTOCOL_BUFFER).get(pv);
+        if (impl == null) { // no match for Protocol AND Version
+          VerProtocolImpl highest = 
+              server.getHighestSupportedProtocol(RpcKind.RPC_PROTOCOL_BUFFER, 
+                  protoName);
+          if (highest == null) {
+            throw new IOException("Unknown protocol: " + protoName);
+          }
+          // protocol supported but not the version that client wants
+          throw new RPC.VersionMismatch(protoName, version,
+              highest.version);
+        }
+        return impl;
+      }
 
       @Override 
       /**
@@ -409,21 +427,8 @@ public class ProtobufRpcEngine implement
         if (server.verbose)
           LOG.info("Call: protocol=" + protocol + ", method=" + methodName);
         
-        ProtoNameVer pv = new ProtoNameVer(protoName, clientVersion);
-        ProtoClassProtoImpl protocolImpl = 
-            server.getProtocolImplMap(RpcKind.RPC_PROTOCOL_BUFFER).get(pv);
-        if (protocolImpl == null) { // no match for Protocol AND Version
-          VerProtocolImpl highest = 
-              server.getHighestSupportedProtocol(RpcKind.RPC_PROTOCOL_BUFFER, 
-                  protoName);
-          if (highest == null) {
-            throw new IOException("Unknown protocol: " + protoName);
-          }
-          // protocol supported but not the version that client wants
-          throw new RPC.VersionMismatch(protoName, clientVersion,
-              highest.version);
-        }
-        
+        ProtoClassProtoImpl protocolImpl = getProtocolImpl(server, protoName,
+            clientVersion);
         BlockingService service = (BlockingService) protocolImpl.protocolImpl;
         MethodDescriptor methodDescriptor = service.getDescriptorForType()
             .findMethodByName(methodName);
@@ -438,7 +443,19 @@ public class ProtobufRpcEngine implement
             .mergeFrom(rpcRequest.getRequest()).build();
         Message result;
         try {
+          long startTime = System.currentTimeMillis();
+          server.rpcDetailedMetrics.init(protocolImpl.protocolClass);
           result = service.callBlockingMethod(methodDescriptor, null, param);
+          int processingTime = (int) (System.currentTimeMillis() - startTime);
+          int qTime = (int) (startTime - receiveTime);
+          if (LOG.isDebugEnabled()) {
+            LOG.info("Served: " + methodName + " queueTime= " + qTime +
+                      " procesingTime= " + processingTime);
+          }
+          server.rpcMetrics.addRpcQueueTime(qTime);
+          server.rpcMetrics.addRpcProcessingTime(processingTime);
+          server.rpcDetailedMetrics.addProcessingTime(methodName,
+              processingTime);
         } catch (ServiceException e) {
           Throwable cause = e.getCause();
           return handleException(cause != null ? cause : e);

Modified: hadoop/common/branches/branch-0.23-PB/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java?rev=1293344&r1=1293343&r2=1293344&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23-PB/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
(original)
+++ hadoop/common/branches/branch-0.23-PB/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
Fri Feb 24 17:14:12 2012
@@ -300,10 +300,15 @@ public abstract class Server {
    * Returns a handle to the rpcMetrics (required in tests)
    * @return rpc metrics
    */
+  @VisibleForTesting
   public RpcMetrics getRpcMetrics() {
     return rpcMetrics;
   }
 
+  @VisibleForTesting
+  public RpcDetailedMetrics getRpcDetailedMetrics() {
+    return rpcDetailedMetrics;
+  }
   
   @VisibleForTesting
   Iterable<? extends Thread> getHandlers() {

Modified: hadoop/common/branches/branch-0.23-PB/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestProtoBufRpc.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestProtoBufRpc.java?rev=1293344&r1=1293343&r2=1293344&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23-PB/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestProtoBufRpc.java
(original)
+++ hadoop/common/branches/branch-0.23-PB/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestProtoBufRpc.java
Fri Feb 24 17:14:12 2012
@@ -17,6 +17,9 @@
  */
 package org.apache.hadoop.ipc;
 
+import static org.apache.hadoop.test.MetricsAsserts.getMetrics;
+import static org.apache.hadoop.test.MetricsAsserts.assertCounterGt;
+
 import java.io.IOException;
 import java.net.InetSocketAddress;
 
@@ -28,6 +31,7 @@ import org.apache.hadoop.ipc.protobuf.Te
 import org.apache.hadoop.ipc.protobuf.TestProtos.EmptyResponseProto;
 import org.apache.hadoop.ipc.protobuf.TestRpcServiceProtos.TestProtobufRpcProto;
 import org.apache.hadoop.ipc.protobuf.TestRpcServiceProtos.TestProtobufRpc2Proto;
+import org.apache.hadoop.metrics2.MetricsRecordBuilder;
 import org.apache.hadoop.net.NetUtils;
 import org.junit.Assert;
 import org.junit.Test;
@@ -187,5 +191,14 @@ public class TestProtoBufRpc {
         .setMessage("hello").build();
     EchoResponseProto echoResponse = client.echo2(null, echoRequest);
     Assert.assertEquals(echoResponse.getMessage(), "hello");
+    
+    // Ensure RPC metrics are updated
+    MetricsRecordBuilder rpcMetrics = getMetrics(server.getRpcMetrics().name());
+    assertCounterGt("RpcQueueTimeNumOps", 0L, rpcMetrics);
+    assertCounterGt("RpcProcessingTimeNumOps", 0L, rpcMetrics);
+    
+    MetricsRecordBuilder rpcDetailedMetrics = 
+        getMetrics(server.getRpcDetailedMetrics().name());
+    assertCounterGt("Echo2NumOps", 0L, rpcDetailedMetrics);
   }
 }
\ No newline at end of file



Mime
View raw message