hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From d...@apache.org
Subject svn commit: r1138317 - in /hadoop/common/branches/MR-279/mapreduce: ./ mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/launcher/ mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/a...
Date Wed, 22 Jun 2011 06:44:54 GMT
Author: ddas
Date: Wed Jun 22 06:44:54 2011
New Revision: 1138317

URL: http://svn.apache.org/viewvc?rev=1138317&view=rev
Log:
Fix for ConcurrentModification exception while iterating through tokens in a UGI in ContainerLauncherImpl.
Contributed by Devaraj Das.

Modified:
    hadoop/common/branches/MR-279/mapreduce/CHANGES.txt
    hadoop/common/branches/MR-279/mapreduce/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/launcher/ContainerLauncherImpl.java
    hadoop/common/branches/MR-279/mapreduce/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java

Modified: hadoop/common/branches/MR-279/mapreduce/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279/mapreduce/CHANGES.txt?rev=1138317&r1=1138316&r2=1138317&view=diff
==============================================================================
--- hadoop/common/branches/MR-279/mapreduce/CHANGES.txt (original)
+++ hadoop/common/branches/MR-279/mapreduce/CHANGES.txt Wed Jun 22 06:44:54 2011
@@ -5,6 +5,9 @@ Trunk (unreleased changes)
 
     MAPREDUCE-279
 
+    Fix for ConcurrentModification exception while iterating through tokens in
+    a UGI in ContainerLauncherImpl. (ddas)
+
     MAPREDUCE-2611. Fix counters, finish times etc. in job history.
     (Siddharth Seth via llu)
 

Modified: hadoop/common/branches/MR-279/mapreduce/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/launcher/ContainerLauncherImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279/mapreduce/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/launcher/ContainerLauncherImpl.java?rev=1138317&r1=1138316&r2=1138317&view=diff
==============================================================================
--- hadoop/common/branches/MR-279/mapreduce/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/launcher/ContainerLauncherImpl.java
(original)
+++ hadoop/common/branches/MR-279/mapreduce/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/launcher/ContainerLauncherImpl.java
Wed Jun 22 06:44:54 2011
@@ -20,6 +20,8 @@ package org.apache.hadoop.mapreduce.v2.a
 
 import java.io.IOException;
 import java.security.PrivilegedAction;
+import java.util.HashMap;
+import java.util.Map;
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.LinkedBlockingQueue;
 import java.util.concurrent.ThreadPoolExecutor;
@@ -71,6 +73,10 @@ public class ContainerLauncherImpl exten
   private BlockingQueue<ContainerLauncherEvent> eventQueue =
       new LinkedBlockingQueue<ContainerLauncherEvent>();
   private RecordFactory recordFactory;
+  //have a cache/map of UGIs so as to avoid creating too many RPC
+  //client connection objects to the same NodeManager
+  private Map<String, UserGroupInformation> ugiMap = 
+    new HashMap<String, UserGroupInformation>();
 
   public ContainerLauncherImpl(AppContext context) {
     super(ContainerLauncherImpl.class.getName());
@@ -130,15 +136,23 @@ public class ContainerLauncherImpl exten
       final String containerManagerBindAddr, ContainerToken containerToken)
       throws IOException {
 
-    UserGroupInformation user = UserGroupInformation.getLoginUser();
+    UserGroupInformation user = UserGroupInformation.getCurrentUser();
+
     if (UserGroupInformation.isSecurityEnabled()) {
-      Token<ContainerTokenIdentifier> token =
+      if(!ugiMap.containsKey(containerManagerBindAddr)) {
+        Token<ContainerTokenIdentifier> token =
           new Token<ContainerTokenIdentifier>(
               containerToken.getIdentifier().array(),
               containerToken.getPassword().array(), new Text(
                   containerToken.getKind()), new Text(
-                  containerToken.getService()));
-      user.addToken(token);
+                      containerToken.getService()));
+        //the user in createRemoteUser in this context is not important
+        user = UserGroupInformation.createRemoteUser(containerManagerBindAddr);
+        user.addToken(token);
+        ugiMap.put(containerManagerBindAddr, user);
+      } else {
+        user = ugiMap.get(containerManagerBindAddr);    
+      }
     }
     ContainerManager proxy =
         user.doAs(new PrivilegedAction<ContainerManager>() {

Modified: hadoop/common/branches/MR-279/mapreduce/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279/mapreduce/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java?rev=1138317&r1=1138316&r2=1138317&view=diff
==============================================================================
--- hadoop/common/branches/MR-279/mapreduce/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
(original)
+++ hadoop/common/branches/MR-279/mapreduce/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
Wed Jun 22 06:44:54 2011
@@ -536,7 +536,7 @@ public class RMContainerAllocator extend
       containersAllocated += allocatedContainers.size();
       while (it.hasNext()) {
         Container allocated = it.next();
-        LOG.info("Assiging container " + allocated);
+        LOG.info("Assigning container " + allocated);
         ContainerRequest assigned = assign(allocated);
           
         if (assigned != null) {



Mime
View raw message