giraph-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From m...@apache.org
Subject git commit: updated refs/heads/trunk to b9f4142
Date Wed, 04 Mar 2015 18:40:46 GMT
Repository: giraph
Updated Branches:
  refs/heads/trunk 0dab32741 -> b9f414296


GIRAPH-996: Large requests degrade performance. Print out warnings. (dlogothetis via majakabiljo)


Project: http://git-wip-us.apache.org/repos/asf/giraph/repo
Commit: http://git-wip-us.apache.org/repos/asf/giraph/commit/b9f41429
Tree: http://git-wip-us.apache.org/repos/asf/giraph/tree/b9f41429
Diff: http://git-wip-us.apache.org/repos/asf/giraph/diff/b9f41429

Branch: refs/heads/trunk
Commit: b9f41429619590e8e1c06d47c2a8268fa3bc07db
Parents: 0dab327
Author: Maja Kabiljo <majakabiljo@fb.com>
Authored: Wed Mar 4 10:39:41 2015 -0800
Committer: Maja Kabiljo <majakabiljo@fb.com>
Committed: Wed Mar 4 10:39:41 2015 -0800

----------------------------------------------------------------------
 CHANGELOG                                                 |  2 ++
 .../java/org/apache/giraph/comm/netty/NettyClient.java    | 10 ++++++++++
 .../main/java/org/apache/giraph/conf/GiraphConstants.java |  7 +++++++
 3 files changed, 19 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/giraph/blob/b9f41429/CHANGELOG
----------------------------------------------------------------------
diff --git a/CHANGELOG b/CHANGELOG
index 331bed6..3c31eb0 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -1,6 +1,8 @@
 Giraph Change Log
 
 Release 1.2.0 - unreleased
+  GIRAPH-996: Large requests degrade performance. Print out warnings. (dlogothetis via majakabiljo)
+
   GIRAPH-990. Current trunk will build for hadoop 1.2.0 not 0.20.203 as stated by documentation
 
   GIRAPH-992: Zookeeper logs have too many NodeExists (edunov)

http://git-wip-us.apache.org/repos/asf/giraph/blob/b9f41429/giraph-core/src/main/java/org/apache/giraph/comm/netty/NettyClient.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/comm/netty/NettyClient.java b/giraph-core/src/main/java/org/apache/giraph/comm/netty/NettyClient.java
index 78e318e..40c6ab9 100644
--- a/giraph-core/src/main/java/org/apache/giraph/comm/netty/NettyClient.java
+++ b/giraph-core/src/main/java/org/apache/giraph/comm/netty/NettyClient.java
@@ -159,6 +159,8 @@ public class NettyClient implements ResetSuperstepMetricsObserver {
   private final int receiveBufferSize;
   /** Do we have a limit on number of open requests */
   private final boolean limitNumberOfOpenRequests;
+  /** Warn if request size is bigger than the buffer size by this factor */
+  private final float requestSizeWarningThreshold;
   /** Maximum number of requests without confirmation we can have */
   private final int maxNumberOfOpenRequests;
   /**
@@ -221,6 +223,8 @@ public class NettyClient implements ResetSuperstepMetricsObserver {
     this.channelsPerServer = GiraphConstants.CHANNELS_PER_SERVER.get(conf);
     sendBufferSize = CLIENT_SEND_BUFFER_SIZE.get(conf);
     receiveBufferSize = CLIENT_RECEIVE_BUFFER_SIZE.get(conf);
+    this.requestSizeWarningThreshold =
+        GiraphConstants.REQUEST_SIZE_WARNING_THRESHOLD.get(conf);
 
     limitNumberOfOpenRequests = LIMIT_NUMBER_OF_OPEN_REQUESTS.get(conf);
     if (limitNumberOfOpenRequests) {
@@ -728,6 +732,12 @@ public class NettyClient implements ResetSuperstepMetricsObserver {
           "request info of " + oldRequestInfo);
       }
     }
+    if (request.getSerializedSize() >
+      requestSizeWarningThreshold * sendBufferSize) {
+      LOG.warn("Creating large request of type " + request.getClass() +
+        ", size " + request.getSerializedSize() +
+        " bytes. Check netty buffer size.");
+    }
     ChannelFuture writeFuture = channel.write(request);
     newRequestInfo.setWriteFuture(writeFuture);
     writeFuture.addListener(logErrorListener);

http://git-wip-us.apache.org/repos/asf/giraph/blob/b9f41429/giraph-core/src/main/java/org/apache/giraph/conf/GiraphConstants.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/conf/GiraphConstants.java b/giraph-core/src/main/java/org/apache/giraph/conf/GiraphConstants.java
index f1cf520..65b7892 100644
--- a/giraph-core/src/main/java/org/apache/giraph/conf/GiraphConstants.java
+++ b/giraph-core/src/main/java/org/apache/giraph/conf/GiraphConstants.java
@@ -732,6 +732,13 @@ public interface GiraphConstants {
           "request size is M, and a worker has P partitions, than its " +
           "initial partition buffer size will be (M / P) * (1 + A).");
 
+
+  /** Warn if msg request size exceeds default size by this factor */
+  FloatConfOption REQUEST_SIZE_WARNING_THRESHOLD = new FloatConfOption(
+      "giraph.msgRequestWarningThreshold", 2.0f,
+      "If request sizes are bigger than the buffer size by this factor " +
+      "warnings are printed to the log and to the command line");
+
   /** Maximum size of vertices (in bytes) per peer before flush */
   IntConfOption MAX_VERTEX_REQUEST_SIZE =
       new IntConfOption("giraph.vertexRequestSize", 512 * ONE_KB,


Mime
View raw message