hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From omal...@apache.org
Subject svn commit: r1079240 - in /hadoop/mapreduce/branches/yahoo-merge/src: contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/ contrib/gridmix/src/test/org/apache/hadoop/mapred/gridmix/ docs/src/documentation/content/xdocs/
Date Tue, 08 Mar 2011 05:59:22 GMT
Author: omalley
Date: Tue Mar  8 05:59:22 2011
New Revision: 1079240

URL: http://svn.apache.org/viewvc?rev=1079240&view=rev
Log:
commit fe88ef0e0c14fb683463de1799c22d03a1f5050e
Author: Ravi Gummadi <gravi@yahoo-inc.com>
Date:   Tue Jan 11 20:24:51 2011 +0530

    : Fix a bug in RoundRobinUserResolver sothat each user from
    trace is mapped to a single proxy user. Also makes it nonmandatory for
    users-list-file to contain groups. Patch is available at
     (gravi)

Modified:
    hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/RoundRobinUserResolver.java
    hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/org/apache/hadoop/mapred/gridmix/TestUserResolve.java
    hadoop/mapreduce/branches/yahoo-merge/src/docs/src/documentation/content/xdocs/gridmix.xml

Modified: hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/RoundRobinUserResolver.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/RoundRobinUserResolver.java?rev=1079240&r1=1079239&r2=1079240&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/RoundRobinUserResolver.java
(original)
+++ hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/RoundRobinUserResolver.java
Tue Mar  8 05:59:22 2011
@@ -38,12 +38,18 @@ public class RoundRobinUserResolver impl
 
   private int uidx = 0;
   private List<UserGroupInformation> users = Collections.emptyList();
-  private final HashMap<UserGroupInformation,UserGroupInformation> usercache =
-    new HashMap<UserGroupInformation,UserGroupInformation>();
+
+  /**
+   *  Mapping between user names of original cluster and UGIs of proxy users of
+   *  simulated cluster
+   */
+  private final HashMap<String,UserGroupInformation> usercache =
+      new HashMap<String,UserGroupInformation>();
   
   /**
-   * Userlist assumes one UGI per line, each UGI matching
-   * &lt;username&gt;,&lt;group&gt;[,group]*
+   * Userlist assumes one user per line.
+   * Each line in users-list-file is of the form &lt;username&gt;[,group]* 
+   * <br> Group names are ignored(they are not read at all).
    */
   private List<UserGroupInformation> parseUserList(URI userUri, 
                                                    Configuration conf) 
@@ -53,47 +59,53 @@ public class RoundRobinUserResolver impl
     }
     
     final Path userloc = new Path(userUri.toString());
-    final Text rawUgi = new Text();
+    final Text rawUgi = new Text();// is of the form   username[,group]*
     final FileSystem fs = userloc.getFileSystem(conf);
-    final ArrayList<UserGroupInformation> ret = new ArrayList();
+    final ArrayList<UserGroupInformation> ugiList =
+        new ArrayList<UserGroupInformation>();
 
     LineReader in = null;
     try {
-      final ArrayList<String> groups = new ArrayList();
       in = new LineReader(fs.open(userloc));
       while (in.readLine(rawUgi) > 0) {
+        // e is end position of user name in this line
         int e = rawUgi.find(",");
-        if (e <= 0) {
+        if (rawUgi.getLength() == 0 || e == 0) {
           throw new IOException("Missing username: " + rawUgi);
         }
+        if (e == -1) {
+          e = rawUgi.getLength();
+        }
         final String username = Text.decode(rawUgi.getBytes(), 0, e);
-        int s = e;
-        while ((e = rawUgi.find(",", ++s)) != -1) {
-          groups.add(Text.decode(rawUgi.getBytes(), s, e - s));
-          s = e;
+        UserGroupInformation ugi = null;
+        try {
+          ugi = UserGroupInformation.createProxyUser(
+            username, UserGroupInformation.getLoginUser());
+        } catch (IOException ioe) {
+          LOG.error("Error while creating a proxy user " ,ioe);
         }
-        groups.add(Text.decode(rawUgi.getBytes(), s, rawUgi.getLength() - s));
-        if (groups.size() == 0) {
-          throw new IOException("Missing groups: " + rawUgi);
+        if (ugi != null) {
+          ugiList.add(ugi);
         }
-        ret.add(UserGroupInformation.createRemoteUser(username));
+        // No need to read groups, even if they exist. Go to next line
       }
     } finally {
       if (in != null) {
         in.close();
       }
     }
-    return ret;
+    return ugiList;
   }
 
   @Override
   public synchronized boolean setTargetUsers(URI userloc, Configuration conf)
   throws IOException {
+    uidx = 0;
     users = parseUserList(userloc, conf);
     if (users.size() == 0) {
       throw new IOException(buildEmptyUsersErrorMsg(userloc));
     }
-    usercache.keySet().retainAll(users);
+    usercache.clear();
     return true;
   }
 
@@ -105,20 +117,13 @@ public class RoundRobinUserResolver impl
   @Override
   public synchronized UserGroupInformation getTargetUgi(
     UserGroupInformation ugi) {
-    UserGroupInformation ret = usercache.get(ugi);
-    if (null == ret) {
-      ret = users.get(uidx++ % users.size());
-      usercache.put(ugi, ret);
-    }
-    UserGroupInformation val = null;
-    try {
-      val = 
-        UserGroupInformation.createProxyUser(ret.getUserName(), 
-          UserGroupInformation.getLoginUser());
-    } catch (IOException e) {
-      LOG.error("Error while creating the proxy user " ,e);
+    // UGI of proxy user
+    UserGroupInformation targetUGI = usercache.get(ugi.getUserName());
+    if (null == targetUGI) {
+      targetUGI = users.get(uidx++ % users.size());
+      usercache.put(ugi.getUserName(), targetUGI);
     }
-    return val;
+    return targetUGI;
   }
 
   /**

Modified: hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/org/apache/hadoop/mapred/gridmix/TestUserResolve.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/org/apache/hadoop/mapred/gridmix/TestUserResolve.java?rev=1079240&r1=1079239&r2=1079240&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/org/apache/hadoop/mapred/gridmix/TestUserResolve.java
(original)
+++ hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/org/apache/hadoop/mapred/gridmix/TestUserResolve.java
Tue Mar  8 05:59:22 2011
@@ -115,27 +115,47 @@ public class TestUserResolve {
         RoundRobinUserResolver.buildEmptyUsersErrorMsg(userRsrc);
     validateBadUsersFile(rslv, userRsrc, expectedErrorMsg);
 
-    // Create user resource file with valid content
+    // Create user resource file with valid content(older users list file)
     writeUserList(usersFilePath,
-        "user0,groupA,groupB,groupC\nuser1,groupA,groupC\n"
-        + "user2,groupB\nuser3,groupA,groupB,groupC\n");
+        "user0,groupA,groupB,groupC\nuser1,groupA,groupC");
+    validateValidUsersFile(rslv, userRsrc);
+
+    // Create user resource file with valid content
+    writeUserList(usersFilePath, "user0,groupA,groupB\nuser1,");
+    validateValidUsersFile(rslv, userRsrc);
 
-    // Validate RoundRobinUserResolver for the case of
-    // user resource file with valid content.
-    assertTrue(rslv.setTargetUsers(new URI(usersFilePath.toString()), conf));
+    // Create user resource file with valid content
+    writeUserList(usersFilePath, "user0\nuser1");
+    validateValidUsersFile(rslv, userRsrc);
+  }
+
+  // Validate RoundRobinUserResolver for the case of
+  // user resource file with valid content.
+  private void validateValidUsersFile(UserResolver rslv, URI userRsrc)
+      throws IOException {
+    assertTrue(rslv.setTargetUsers(userRsrc, conf));
     UserGroupInformation ugi1 = UserGroupInformation.createRemoteUser("hfre0");
     assertEquals("user0", rslv.getTargetUgi(ugi1).getUserName());
     assertEquals("user1", 
       rslv.getTargetUgi(UserGroupInformation.createRemoteUser("hfre1"))
           .getUserName());
-    assertEquals("user2", 
+    assertEquals("user0",
       rslv.getTargetUgi(UserGroupInformation.createRemoteUser("hfre2"))
           .getUserName());
     assertEquals("user0", rslv.getTargetUgi(ugi1).getUserName());
-    assertEquals("user3", 
+    assertEquals("user1",
       rslv.getTargetUgi(UserGroupInformation.createRemoteUser("hfre3"))
           .getUserName());
-    assertEquals("user0", rslv.getTargetUgi(ugi1).getUserName());
+    // Verify if same user comes again, its mapped user name should be
+    // correct even though UGI is constructed again.
+    assertEquals("user0", rslv.getTargetUgi(
+        UserGroupInformation.createRemoteUser("hfre0")).getUserName());
+    assertEquals("user0",
+        rslv.getTargetUgi(UserGroupInformation.createRemoteUser("hfre5"))
+            .getUserName());
+    assertEquals("user0",
+        rslv.getTargetUgi(UserGroupInformation.createRemoteUser("hfre0"))
+            .getUserName());
   }
 
   @Test

Modified: hadoop/mapreduce/branches/yahoo-merge/src/docs/src/documentation/content/xdocs/gridmix.xml
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/docs/src/documentation/content/xdocs/gridmix.xml?rev=1079240&r1=1079239&r2=1079240&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/src/docs/src/documentation/content/xdocs/gridmix.xml
(original)
+++ hadoop/mapreduce/branches/yahoo-merge/src/docs/src/documentation/content/xdocs/gridmix.xml
Tue Mar  8 05:59:22 2011
@@ -480,28 +480,27 @@ hadoop jar &lt;gridmix-jar&gt; org.apach
       </table>
       <p>If the parameter <code>gridmix.user.resolve.class</code> is set
to
       <code>org.apache.hadoop.mapred.gridmix.RoundRobinUserResolver</code>,
-      we need to define a users-list file with a list of test users and groups.
+      we need to define a users-list file with a list of test users.
       This is specified using the <code>-users</code> option to GridMix.</p>
       <note>
       Specifying a users-list file using the <code>-users</code> option is
       mandatory when using the round-robin user-resolver. Other user-resolvers
       ignore this option.
       </note>
-      <p>A users-list file has one user-group-information (UGI) per line, each
-      UGI of the format:</p>
+      <p>A users-list file has one user per line, each line of the format:</p>
       <source>
-      &lt;username&gt;,&lt;group&gt;[,group]*
+      &lt;username&gt;
       </source>
       <p>For example:</p>
       <source>
-      user1,group1
-      user2,group2,group3
-      user3,group3,group4
+      user1
+      user2
+      user3
       </source>
       <p>In the above example we have defined three users <code>user1</code>,
-      <code>user2</code> and <code>user3</code> with their respective
groups.
+      <code>user2</code> and <code>user3</code>.
       Now we would associate each unique user in the trace to the above users
-      defined in round-robin fashion. For example, if traces users are
+      defined in round-robin fashion. For example, if trace's users are
       <code>tuser1</code>, <code>tuser2</code>, <code>tuser3</code>,
       <code>tuser4</code> and <code>tuser5</code>, then the mappings
would
       be:</p>



Mime
View raw message