hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sur...@apache.org
Subject svn commit: r893667 - in /hadoop/common/branches/branch-0.21: CHANGES.txt src/java/org/apache/hadoop/ipc/Server.java src/test/core/org/apache/hadoop/ipc/TestIPCServerResponder.java
Date Thu, 24 Dec 2009 00:50:09 GMT
Author: suresh
Date: Thu Dec 24 00:50:08 2009
New Revision: 893667

URL: http://svn.apache.org/viewvc?rev=893667&view=rev
Log:
HADOOP-6460. Merging change 893666 from trunk to 21 -Reinitializes buffers used for serializing
responses in ipc server on exceeding maximum response size to free up Java heap. Contributed
by Suresh Srinivas.

Modified:
    hadoop/common/branches/branch-0.21/CHANGES.txt
    hadoop/common/branches/branch-0.21/src/java/org/apache/hadoop/ipc/Server.java
    hadoop/common/branches/branch-0.21/src/test/core/org/apache/hadoop/ipc/TestIPCServerResponder.java

Modified: hadoop/common/branches/branch-0.21/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.21/CHANGES.txt?rev=893667&r1=893666&r2=893667&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.21/CHANGES.txt (original)
+++ hadoop/common/branches/branch-0.21/CHANGES.txt Thu Dec 24 00:50:08 2009
@@ -1161,6 +1161,10 @@
     HADOOP-6269. Fix threading issue with defaultResource in Configuration.
     (Sreekanth Ramakrishnan via cdouglas)
 
+    HADOOP-6460. Reinitializes buffers used for serializing responses in ipc
+    server on exceeding maximum response size to free up Java heap. (suresh)
+    
+
 Release 0.20.1 - 2009-09-01
 
   INCOMPATIBLE CHANGES

Modified: hadoop/common/branches/branch-0.21/src/java/org/apache/hadoop/ipc/Server.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.21/src/java/org/apache/hadoop/ipc/Server.java?rev=893667&r1=893666&r2=893667&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.21/src/java/org/apache/hadoop/ipc/Server.java (original)
+++ hadoop/common/branches/branch-0.21/src/java/org/apache/hadoop/ipc/Server.java Thu Dec
24 00:50:08 2009
@@ -90,6 +90,12 @@
    */
   private static final int MAX_QUEUE_SIZE_PER_HANDLER = 100;
   
+  /**
+   * Initial and max size of response buffer
+   */
+  static int INITIAL_RESP_BUF_SIZE = 10240;
+  static int MAX_RESP_BUF_SIZE = 1024*1024;
+    
   public static final Log LOG = LogFactory.getLog(Server.class);
 
   private static final ThreadLocal<Server> SERVER = new ThreadLocal<Server>();
@@ -938,7 +944,8 @@
     public void run() {
       LOG.info(getName() + ": starting");
       SERVER.set(Server.this);
-      ByteArrayOutputStream buf = new ByteArrayOutputStream(10240);
+      ByteArrayOutputStream buf = 
+        new ByteArrayOutputStream(INITIAL_RESP_BUF_SIZE);
       while (running) {
         try {
           final Call call = callQueue.take(); // pop the queue; maybe blocked here
@@ -979,10 +986,16 @@
             error = StringUtils.stringifyException(e);
           }
           CurCall.set(null);
-
           setupResponse(buf, call, 
                         (error == null) ? Status.SUCCESS : Status.ERROR, 
                         value, errorClass, error);
+          // Discard the large buf and reset it back to 
+          // smaller size to freeup heap
+          if (buf.size() > MAX_RESP_BUF_SIZE) {
+            LOG.warn("Large response size " + buf.size() + " for call " + 
+                call.toString());
+            buf = new ByteArrayOutputStream(INITIAL_RESP_BUF_SIZE);
+          }
           responder.doRespond(call);
         } catch (InterruptedException e) {
           if (running) {                          // unexpected -- log it

Modified: hadoop/common/branches/branch-0.21/src/test/core/org/apache/hadoop/ipc/TestIPCServerResponder.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.21/src/test/core/org/apache/hadoop/ipc/TestIPCServerResponder.java?rev=893667&r1=893666&r2=893667&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.21/src/test/core/org/apache/hadoop/ipc/TestIPCServerResponder.java
(original)
+++ hadoop/common/branches/branch-0.21/src/test/core/org/apache/hadoop/ipc/TestIPCServerResponder.java
Thu Dec 24 00:50:08 2009
@@ -114,6 +114,12 @@
     }
   }
 
+  public void testResponseBuffer() throws Exception {
+    Server.INITIAL_RESP_BUF_SIZE = 1;
+    Server.MAX_RESP_BUF_SIZE = 1;
+    testServerResponder(1, true, 1, 1, 5);
+  }
+
   public void testServerResponder() throws Exception {
     testServerResponder(10, true, 1, 10, 200);
   }



Mime
View raw message