hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ravigumm...@apache.org
Subject svn commit: r1097679 - in /hadoop/mapreduce/trunk: ./ src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/ src/contrib/gridmix/src/test/org/apache/hadoop/mapred/gridmix/ src/docs/src/documentation/content/xdocs/
Date Fri, 29 Apr 2011 03:49:59 GMT
Author: ravigummadi
Date: Fri Apr 29 03:49:58 2011
New Revision: 1097679

URL: http://svn.apache.org/viewvc?rev=1097679&view=rev
Log:
MAPREDUCE-2416. Remove the restriction of specifying group names in users-list file for Gridmix
in RoundRobinUserResolver mode.

Modified:
    hadoop/mapreduce/trunk/CHANGES.txt
    hadoop/mapreduce/trunk/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/RoundRobinUserResolver.java
    hadoop/mapreduce/trunk/src/contrib/gridmix/src/test/org/apache/hadoop/mapred/gridmix/TestUserResolve.java
    hadoop/mapreduce/trunk/src/docs/src/documentation/content/xdocs/gridmix.xml

Modified: hadoop/mapreduce/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/CHANGES.txt?rev=1097679&r1=1097678&r2=1097679&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/CHANGES.txt (original)
+++ hadoop/mapreduce/trunk/CHANGES.txt Fri Apr 29 03:49:58 2011
@@ -83,6 +83,9 @@ Trunk (unreleased changes)
 
   BUG FIXES
 
+    MAPREDUCE-2416. Remove the restriction of specifying group names in
+    users-list file for Gridmix in RoundRobinUserResolver mode.
+
     MAPREDUCE-2417. Fix Gridmix in RoundRobinUserResolver mode to
     map testing/proxy users to unique users in a trace.
 

Modified: hadoop/mapreduce/trunk/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/RoundRobinUserResolver.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/RoundRobinUserResolver.java?rev=1097679&r1=1097678&r2=1097679&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/RoundRobinUserResolver.java
(original)
+++ hadoop/mapreduce/trunk/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/RoundRobinUserResolver.java
Fri Apr 29 03:49:58 2011
@@ -47,8 +47,9 @@ public class RoundRobinUserResolver impl
       new HashMap<String,UserGroupInformation>();
   
   /**
-   * Userlist assumes one UGI per line, each UGI matching
-   * &lt;username&gt;,&lt;group&gt;[,group]*
+   * Userlist assumes one user per line.
+   * Each line in users-list-file is of the form &lt;username&gt;[,group]* 
+   * <br> Group names are ignored(they are not parsed at all).
    */
   private List<UserGroupInformation> parseUserList(URI userUri, 
                                                    Configuration conf) 
@@ -60,35 +61,40 @@ public class RoundRobinUserResolver impl
     final Path userloc = new Path(userUri.toString());
     final Text rawUgi = new Text();
     final FileSystem fs = userloc.getFileSystem(conf);
-    final ArrayList<UserGroupInformation> ret = new ArrayList();
+    final ArrayList<UserGroupInformation> ugiList =
+        new ArrayList<UserGroupInformation>();
 
     LineReader in = null;
     try {
-      final ArrayList<String> groups = new ArrayList();
       in = new LineReader(fs.open(userloc));
-      while (in.readLine(rawUgi) > 0) {
+      while (in.readLine(rawUgi) > 0) {//line is of the form username[,group]*
+        // e is end position of user name in this line
         int e = rawUgi.find(",");
-        if (e <= 0) {
+        if (rawUgi.getLength() == 0 || e == 0) {
           throw new IOException("Missing username: " + rawUgi);
         }
+        if (e == -1) {
+          e = rawUgi.getLength();
+        }
         final String username = Text.decode(rawUgi.getBytes(), 0, e);
-        int s = e;
-        while ((e = rawUgi.find(",", ++s)) != -1) {
-          groups.add(Text.decode(rawUgi.getBytes(), s, e - s));
-          s = e;
+        UserGroupInformation ugi = null;
+        try {
+          ugi = UserGroupInformation.createProxyUser(username,
+                    UserGroupInformation.getLoginUser());
+        } catch (IOException ioe) {
+          LOG.error("Error while creating a proxy user " ,ioe);
         }
-        groups.add(Text.decode(rawUgi.getBytes(), s, rawUgi.getLength() - s));
-        if (groups.size() == 0) {
-          throw new IOException("Missing groups: " + rawUgi);
+        if (ugi != null) {
+          ugiList.add(ugi);
         }
-        ret.add(UserGroupInformation.createRemoteUser(username));
+        // No need to parse groups, even if they exist. Go to next line
       }
     } finally {
       if (in != null) {
         in.close();
       }
     }
-    return ret;
+    return ugiList;
   }
 
   @Override
@@ -113,7 +119,7 @@ public class RoundRobinUserResolver impl
     UserGroupInformation ugi) {
     // UGI of proxy user
     UserGroupInformation targetUGI = usercache.get(ugi.getUserName());
-    if (null == targetUGI) {
+    if (targetUGI == null) {
       targetUGI = users.get(uidx++ % users.size());
       usercache.put(ugi.getUserName(), targetUGI);
     }

Modified: hadoop/mapreduce/trunk/src/contrib/gridmix/src/test/org/apache/hadoop/mapred/gridmix/TestUserResolve.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/gridmix/src/test/org/apache/hadoop/mapred/gridmix/TestUserResolve.java?rev=1097679&r1=1097678&r2=1097679&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/gridmix/src/test/org/apache/hadoop/mapred/gridmix/TestUserResolve.java
(original)
+++ hadoop/mapreduce/trunk/src/contrib/gridmix/src/test/org/apache/hadoop/mapred/gridmix/TestUserResolve.java
Fri Apr 29 03:49:58 2011
@@ -115,10 +115,21 @@ public class TestUserResolve {
         RoundRobinUserResolver.buildEmptyUsersErrorMsg(userRsrc);
     validateBadUsersFile(rslv, userRsrc, expectedErrorMsg);
 
-    // Create user resource file with valid content
+    // Create user resource file with valid content like older users list file
+    // with usernames and groups
     writeUserList(usersFilePath,
         "user0,groupA,groupB,groupC\nuser1,groupA,groupC\n");
     validateValidUsersFile(rslv, userRsrc);
+
+    // Create user resource file with valid content with
+    // usernames with groups and without groups
+    writeUserList(usersFilePath, "user0,groupA,groupB\nuser1,");
+    validateValidUsersFile(rslv, userRsrc);
+
+    // Create user resource file with valid content with
+    // usernames without groups
+    writeUserList(usersFilePath, "user0\nuser1");
+    validateValidUsersFile(rslv, userRsrc);
   }
 
   // Validate RoundRobinUserResolver for the case of
@@ -129,15 +140,16 @@ public class TestUserResolve {
     UserGroupInformation ugi1 = UserGroupInformation.createRemoteUser("hfre0");
     assertEquals("user0", rslv.getTargetUgi(ugi1).getUserName());
     assertEquals("user1", 
-      rslv.getTargetUgi(UserGroupInformation.createRemoteUser("hfre1"))
-          .getUserName());
+        rslv.getTargetUgi(UserGroupInformation.createRemoteUser("hfre1"))
+            .getUserName());
     assertEquals("user0",
-      rslv.getTargetUgi(UserGroupInformation.createRemoteUser("hfre2"))
-          .getUserName());
+        rslv.getTargetUgi(UserGroupInformation.createRemoteUser("hfre2"))
+            .getUserName());
     assertEquals("user0", rslv.getTargetUgi(ugi1).getUserName());
     assertEquals("user1",
-      rslv.getTargetUgi(UserGroupInformation.createRemoteUser("hfre3"))
-          .getUserName());
+        rslv.getTargetUgi(UserGroupInformation.createRemoteUser("hfre3"))
+            .getUserName());
+
     // Verify if same user comes again, its mapped user name should be
     // correct even though UGI is constructed again.
     assertEquals("user0", rslv.getTargetUgi(

Modified: hadoop/mapreduce/trunk/src/docs/src/documentation/content/xdocs/gridmix.xml
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/docs/src/documentation/content/xdocs/gridmix.xml?rev=1097679&r1=1097678&r2=1097679&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/docs/src/documentation/content/xdocs/gridmix.xml (original)
+++ hadoop/mapreduce/trunk/src/docs/src/documentation/content/xdocs/gridmix.xml Fri Apr 29
03:49:58 2011
@@ -472,28 +472,27 @@ hadoop jar &lt;gridmix-jar&gt; org.apach
       </table>
       <p>If the parameter <code>gridmix.user.resolve.class</code> is set
to
       <code>org.apache.hadoop.mapred.gridmix.RoundRobinUserResolver</code>,
-      we need to define a users-list file with a list of test users and groups.
+      we need to define a users-list file with a list of test users.
       This is specified using the <code>-users</code> option to GridMix.</p>
       <note>
       Specifying a users-list file using the <code>-users</code> option is
       mandatory when using the round-robin user-resolver. Other user-resolvers
       ignore this option.
       </note>
-      <p>A users-list file has one user-group-information (UGI) per line, each
-      UGI of the format:</p>
+      <p>A users-list file has one user per line, each line of the format:</p>
       <source>
-      &lt;username&gt;,&lt;group&gt;[,group]*
+      &lt;username&gt;
       </source>
       <p>For example:</p>
       <source>
-      user1,group1
-      user2,group2,group3
-      user3,group3,group4
+      user1
+      user2
+      user3
       </source>
       <p>In the above example we have defined three users <code>user1</code>,
-      <code>user2</code> and <code>user3</code> with their respective
groups.
+      <code>user2</code> and <code>user3</code>.
       Now we would associate each unique user in the trace to the above users
-      defined in round-robin fashion. For example, if traces users are
+      defined in round-robin fashion. For example, if trace's users are
       <code>tuser1</code>, <code>tuser2</code>, <code>tuser3</code>,
       <code>tuser4</code> and <code>tuser5</code>, then the mappings
would
       be:</p>
@@ -504,6 +503,10 @@ hadoop jar &lt;gridmix-jar&gt; org.apach
       tuser4 -&gt; user1
       tuser5 -&gt; user2
       </source>
+      <p>For backward compatibility reasons, each line of users-list file can
+      contain username followed by groupnames in the form username[,group]*.
+      The groupnames will be ignored by Gridmix.
+      </p>
     </section>
     <section id="assumptions">
       <title>Simplifying Assumptions</title>



Mime
View raw message