hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cnaur...@apache.org
Subject svn commit: r1511546 - in /hadoop/common/branches/branch-1-win: ./ src/core/org/apache/hadoop/security/ src/core/org/apache/hadoop/util/ src/mapred/org/apache/hadoop/mapred/ src/tools/org/apache/hadoop/tools/
Date Thu, 08 Aug 2013 00:02:38 GMT
Author: cnauroth
Date: Thu Aug  8 00:02:37 2013
New Revision: 1511546

URL: http://svn.apache.org/r1511546
Log:
HADOOP-9790. Job token path is not unquoted properly. Contributed by Xi Fang.

Modified:
    hadoop/common/branches/branch-1-win/CHANGES.branch-1-win.txt
    hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/security/UserGroupInformation.java
    hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/util/Shell.java
    hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/Child.java
    hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/TaskRunner.java
    hadoop/common/branches/branch-1-win/src/tools/org/apache/hadoop/tools/DistCp.java

Modified: hadoop/common/branches/branch-1-win/CHANGES.branch-1-win.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/CHANGES.branch-1-win.txt?rev=1511546&r1=1511545&r2=1511546&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/CHANGES.branch-1-win.txt (original)
+++ hadoop/common/branches/branch-1-win/CHANGES.branch-1-win.txt Thu Aug  8 00:02:37 2013
@@ -323,6 +323,8 @@ Branch-hadoop-1-win (branched from branc
     HADOOP-9609. Remove sh dependency of bin-package target. (Chuan Liu via
     cnauroth)
 
+    HADOOP-9790. Job token path is not unquoted properly. (Xi Fang via cnauroth)
+
   Merged from branch-1
 
     HDFS-385. Backport: Add support for an experimental API that allows a

Modified: hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/security/UserGroupInformation.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/security/UserGroupInformation.java?rev=1511546&r1=1511545&r2=1511546&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/security/UserGroupInformation.java
(original)
+++ hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/security/UserGroupInformation.java
Thu Aug  8 00:02:37 2013
@@ -543,6 +543,7 @@ public class UserGroupInformation {
         if (fileLocation != null && isSecurityEnabled()) {
           // load the token storage file and put all of the tokens into the
           // user.
+          fileLocation = Shell.preprocessEnvVar(fileLocation);
           Credentials cred = Credentials.readTokenStorageFile(
               new Path("file:///" + fileLocation), conf);
           for (Token<?> token: cred.getAllTokens()) {

Modified: hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/util/Shell.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/util/Shell.java?rev=1511546&r1=1511545&r2=1511546&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/util/Shell.java (original)
+++ hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/util/Shell.java Thu Aug
 8 00:02:37 2013
@@ -156,6 +156,14 @@ abstract public class Shell {
     return winUtilsPath;
   }
 
+  public static final String preprocessEnvVar(String env) {
+    /* trim leading and trailing quotes of an environment variable if the OS is Windows */
+    if (Shell.WINDOWS && env != null) {
+      env = env.replaceAll("^\"|\"$", "");
+    }
+    return env;
+  }
+
   /** a Unix command to get the current user's name */
   public final static String USER_NAME_COMMAND = "whoami";
 

Modified: hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/Child.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/Child.java?rev=1511546&r1=1511545&r2=1511546&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/Child.java (original)
+++ hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/Child.java Thu
Aug  8 00:02:37 2013
@@ -90,12 +90,7 @@ class Child {
     // file name is passed thru env
     String jobTokenFile = 
       System.getenv().get(UserGroupInformation.HADOOP_TOKEN_FILE_LOCATION);
-    if(Shell.WINDOWS) {
-      if(jobTokenFile.charAt(0)=='"')
-        jobTokenFile = jobTokenFile.substring(1);
-      if(jobTokenFile.charAt(jobTokenFile.length()-1) == '"')
-        jobTokenFile = jobTokenFile.substring(0, jobTokenFile.length()-1);
-    }
+    jobTokenFile = Shell.preprocessEnvVar(jobTokenFile);
     Credentials credentials = 
       TokenCache.loadTokens(jobTokenFile, defaultConf);
     LOG.debug("loading token. # keys =" +credentials.numberOfSecretKeys() + 

Modified: hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/TaskRunner.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/TaskRunner.java?rev=1511546&r1=1511545&r2=1511546&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/TaskRunner.java
(original)
+++ hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/TaskRunner.java
Thu Aug  8 00:02:37 2013
@@ -515,11 +515,7 @@ abstract class TaskRunner extends Thread
 
     // if temp directory path is not absolute, prepend it with workDir.
     if (!tmpDir.isAbsolute()) {
-      if (Shell.WINDOWS)
-        // trim leading and trailing quotes on Windows
-        tmpDir = new Path(workDir.toString().replaceAll("^\"|\"$", ""), tmp);
-      else
-        tmpDir = new Path(workDir.toString(), tmp);
+      tmpDir =  new Path(Shell.preprocessEnvVar(workDir.toString()), tmp);
       if (createDir) {
         FileSystem localFs = FileSystem.getLocal(conf);
         if (!localFs.mkdirs(tmpDir) && 
@@ -808,11 +804,7 @@ abstract class TaskRunner extends Thread
   private static void symlink(File workDir, String target, String link)
       throws IOException {
     if (link != null) {
-      if (Shell.WINDOWS)
-        // trim leading and trailing quotes on Windows
-        link = workDir.toString().replaceAll("^\"|\"$", "") + File.separator + link;
-      else
-        link = workDir.toString() + File.separator + link;
+      link = Shell.preprocessEnvVar(workDir.toString()) + File.separator + link;
       File flink = new File(link);
       if (!flink.exists()) {
         LOG.info(String.format("Creating symlink: %s <- %s", target, link));

Modified: hadoop/common/branches/branch-1-win/src/tools/org/apache/hadoop/tools/DistCp.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/tools/org/apache/hadoop/tools/DistCp.java?rev=1511546&r1=1511545&r2=1511546&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/src/tools/org/apache/hadoop/tools/DistCp.java (original)
+++ hadoop/common/branches/branch-1-win/src/tools/org/apache/hadoop/tools/DistCp.java Thu
Aug  8 00:02:37 2013
@@ -67,6 +67,7 @@ import org.apache.hadoop.mapred.Sequence
 import org.apache.hadoop.mapreduce.JobSubmissionFiles;
 import org.apache.hadoop.mapreduce.security.TokenCache;
 import org.apache.hadoop.security.AccessControlException;
+import org.apache.hadoop.util.Shell;
 import org.apache.hadoop.util.StringUtils;
 import org.apache.hadoop.util.Tool;
 import org.apache.hadoop.util.ToolRunner;
@@ -982,6 +983,7 @@ public class DistCp implements Tool {
     // Propagate delegation related props to DistCp job
     String tokenFile = System.getenv("HADOOP_TOKEN_FILE_LOCATION");
     if (tokenFile != null) {
+      tokenFile = Shell.preprocessEnvVar(tokenFile);
       LOG
         .info("Setting env property for mapreduce.job.credentials.binary to: "
           + tokenFile);



Mime
View raw message