hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From d...@apache.org
Subject svn commit: r957086 - in /hadoop/mapreduce/trunk: CHANGES.txt src/java/org/apache/hadoop/mapred/JobTracker.java src/java/org/apache/hadoop/mapred/tools/MRAdmin.java src/test/mapred/org/apache/hadoop/security/TestMapredGroupMappingServiceRefresh.java
Date Wed, 23 Jun 2010 00:00:04 GMT
Author: ddas
Date: Wed Jun 23 00:00:04 2010
New Revision: 957086

URL: http://svn.apache.org/viewvc?rev=957086&view=rev
Log:
MAPREDUCE-1855. Makes the refresh methods (for groups and proxy users) independent of the
client side configuration. Contributed by Boris Shkolnik.

Modified:
    hadoop/mapreduce/trunk/CHANGES.txt
    hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java
    hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/tools/MRAdmin.java
    hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/security/TestMapredGroupMappingServiceRefresh.java

Modified: hadoop/mapreduce/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/CHANGES.txt?rev=957086&r1=957085&r2=957086&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/CHANGES.txt (original)
+++ hadoop/mapreduce/trunk/CHANGES.txt Wed Jun 23 00:00:04 2010
@@ -161,6 +161,9 @@ Release 0.21.0 - Unreleased
     MAPREDUCE-1683. Remove JNI calls from ClusterStatus cstr. (Arun Murthy and
     Luke Lu via cdouglas)
 
+    MAPREDUCE-1855. Makes the refresh methods (for groups and proxy users) 
+    independent of the client side configuration. (Boris Shkolnik via ddas)
+
   NEW FEATURES
 
     MAPREDUCE-1774. Large-scale Automated Framework (Sharad Agarwal, Sreekanth

Modified: hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java?rev=957086&r1=957085&r2=957086&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java (original)
+++ hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java Wed Jun 23 00:00:04
2010
@@ -4449,18 +4449,18 @@ public class JobTracker implements MRCon
   }
 
   @Override
-  public void refreshSuperUserGroupsConfiguration(Configuration conf) {
+  public void refreshSuperUserGroupsConfiguration() {
     LOG.info("Refreshing superuser proxy groups mapping ");
 
-    ProxyUsers.refreshSuperUserGroupsConfiguration(conf);
+    ProxyUsers.refreshSuperUserGroupsConfiguration();
   }
 
   @Override
-  public void refreshUserToGroupsMappings(Configuration conf) throws IOException {
+  public void refreshUserToGroupsMappings() throws IOException {
     LOG.info("Refreshing all user-to-groups mappings. Requested by user: " + 
              UserGroupInformation.getCurrentUser().getShortUserName());
     
-    Groups.getUserToGroupsMappingService(conf).refresh();
+    Groups.getUserToGroupsMappingService().refresh();
   }
   
   private boolean perTaskMemoryConfigurationSetOnJT() {

Modified: hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/tools/MRAdmin.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/tools/MRAdmin.java?rev=957086&r1=957085&r2=957086&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/tools/MRAdmin.java (original)
+++ hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/tools/MRAdmin.java Wed Jun 23
00:00:04 2010
@@ -195,7 +195,7 @@ public class MRAdmin extends Configured 
               RefreshUserMappingsProtocol.class));
 
     // Refresh the user-to-groups mappings
-    refreshProtocol.refreshSuperUserGroupsConfiguration(conf);
+    refreshProtocol.refreshSuperUserGroupsConfiguration();
 
     return 0;
   }
@@ -227,7 +227,7 @@ public class MRAdmin extends Configured 
               RefreshUserMappingsProtocol.class));
 
     // Refresh the user-to-groups mappings
-    refreshProtocol.refreshUserToGroupsMappings(conf);
+    refreshProtocol.refreshUserToGroupsMappings();
     
     return 0;
   }

Modified: hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/security/TestMapredGroupMappingServiceRefresh.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/security/TestMapredGroupMappingServiceRefresh.java?rev=957086&r1=957085&r2=957086&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/security/TestMapredGroupMappingServiceRefresh.java
(original)
+++ hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/security/TestMapredGroupMappingServiceRefresh.java
Wed Jun 23 00:00:04 2010
@@ -25,8 +25,13 @@ import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
 
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
 import java.io.IOException;
+import java.io.PrintWriter;
 import java.net.URI;
+import java.net.URL;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
@@ -36,8 +41,10 @@ import org.apache.commons.logging.LogFac
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.CommonConfigurationKeys;
 import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.hdfs.server.namenode.NameNode;
+import org.apache.hadoop.hdfs.tools.DFSAdmin;
 import org.apache.hadoop.mapred.JobConf;
 import org.apache.hadoop.mapred.MiniMRCluster;
 import org.apache.hadoop.mapred.tools.MRAdmin;
@@ -52,6 +59,7 @@ public class TestMapredGroupMappingServi
   private MiniDFSCluster cluster;
   JobConf config;
   private static long groupRefreshTimeoutSec = 2;
+  private String tempResource = null;
   private static final Log LOG = LogFactory
       .getLog(TestMapredGroupMappingServiceRefresh.class);
   
@@ -98,6 +106,7 @@ public class TestMapredGroupMappingServi
       3, null, null, config);
     
     config.set(JTConfig.JT_IPC_ADDRESS, "localhost:"+miniMRCluster.getJobTrackerPort());
+    ProxyUsers.refreshSuperUserGroupsConfiguration(config);
   }
 
   @After
@@ -105,6 +114,10 @@ public class TestMapredGroupMappingServi
     if(cluster!=null) {
       cluster.shutdown();
     }
+    if(tempResource!=null) {
+      File f = new File(tempResource);
+      f.delete();
+    }
   }
   
   @Test
@@ -162,6 +175,7 @@ public class TestMapredGroupMappingServi
     
     config.set(userKeyGroups, "gr3,gr4,gr5"); // superuser can proxy for this group
     config.set(userKeyHosts,"127.0.0.1");
+    ProxyUsers.refreshSuperUserGroupsConfiguration(config);
     
     UserGroupInformation ugi1 = mock(UserGroupInformation.class);
     UserGroupInformation ugi2 = mock(UserGroupInformation.class);
@@ -199,16 +213,16 @@ public class TestMapredGroupMappingServi
       fail("first auth for " + ugi2.getShortUserName() + " should've succeeded: " + e.getLocalizedMessage());
     }
     
+    // refresh will look at configuration on the server side
+    // add additional resource with the new value
+    // so the server side will pick it up
+    String rsrc = "testRefreshSuperUserGroupsConfiguration_rsrc.xml";
+    addNewConfigResource(rsrc, userKeyGroups, "gr2", userKeyHosts, "127.0.0.1");  
+    
     MRAdmin admin = new MRAdmin(config);
     String [] args = new String[]{"-refreshSuperUserGroupsConfiguration"};
-    NameNode nn = cluster.getNameNode();
-    Configuration conf = new Configuration(config);
-    conf.set(userKeyGroups, "gr2"); // superuser can proxy for this group
-    admin.setConf(conf);
     admin.run(args);
     
-    //check after...
-    
     try {
       ProxyUsers.authorize(ugi2, "127.0.0.1", config);
       fail("second auth for " + ugi2.getShortUserName() + " should've failed ");
@@ -224,5 +238,26 @@ public class TestMapredGroupMappingServi
       fail("second auth for " + ugi1.getShortUserName() + " should've succeeded: " + e.getLocalizedMessage());
     }    
   }
+  
+  private void addNewConfigResource(String rsrcName, String keyGroup,
+      String groups, String keyHosts, String hosts)  throws FileNotFoundException {
+    // location for temp resource should be in CLASSPATH
+    URL url = config.getResource("mapred-site.xml");
+    Path p = new Path(url.getPath());
+    Path dir = p.getParent();
+    tempResource = dir.toString() + "/" + rsrcName;
+    
+    
+    String newResource =
+    "<configuration>"+
+    "<property><name>" + keyGroup + "</name><value>"+groups+"</value></property>"
+
+    "<property><name>" + keyHosts + "</name><value>"+hosts+"</value></property>"
+
+    "</configuration>";
+    PrintWriter writer = new PrintWriter(new FileOutputStream(tempResource));
+    writer.println(newResource);
+    writer.close();
+
+    Configuration.addDefaultResource(rsrcName);
+  }
 
 }



Mime
View raw message