hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cnaur...@apache.org
Subject git commit: YARN-2720. Windows: Wildcard classpath variables not expanded against resources contained in archives. Contributed by Craig Welch.
Date Tue, 21 Oct 2014 19:36:19 GMT
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 c800e058e -> a06613427


YARN-2720. Windows: Wildcard classpath variables not expanded against resources contained
in archives. Contributed by Craig Welch.

(cherry picked from commit 6637e3cf95b3a9be8d6b9cd66bc849a0607e8ed5)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a0661342
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a0661342
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a0661342

Branch: refs/heads/branch-2
Commit: a066134277716a9709b63cfb11b4be5627a9bd3f
Parents: c800e05
Author: cnauroth <cnauroth@apache.org>
Authored: Tue Oct 21 12:33:21 2014 -0700
Committer: cnauroth <cnauroth@apache.org>
Committed: Tue Oct 21 12:34:58 2014 -0700

----------------------------------------------------------------------
 .../main/java/org/apache/hadoop/fs/FileUtil.java   | 17 +++++++++++++----
 .../java/org/apache/hadoop/util/Classpath.java     |  2 +-
 .../java/org/apache/hadoop/fs/TestFileUtil.java    |  4 +++-
 hadoop-yarn-project/CHANGES.txt                    |  3 +++
 .../WindowsSecureContainerExecutor.java            |  6 ++++--
 .../containermanager/launcher/ContainerLaunch.java |  6 ++++--
 6 files changed, 28 insertions(+), 10 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0661342/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
index 2b05293..0848cb1 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
@@ -1212,10 +1212,11 @@ public class FileUtil {
    * @param pwd Path to working directory to save jar
    * @param callerEnv Map<String, String> caller's environment variables to use
    *   for expansion
-   * @return String absolute path to new jar
+   * @return String[] with absolute path to new jar in position 0 and
+   *   unexpanded wild card entry path in position 1
    * @throws IOException if there is an I/O error while writing the jar file
    */
-  public static String createJarWithClassPath(String inputClassPath, Path pwd,
+  public static String[] createJarWithClassPath(String inputClassPath, Path pwd,
       Map<String, String> callerEnv) throws IOException {
     // Replace environment variables, case-insensitive on Windows
     @SuppressWarnings("unchecked")
@@ -1235,6 +1236,7 @@ public class FileUtil {
       LOG.debug("mkdirs false for " + workingDir + ", execution will continue");
     }
 
+    StringBuilder unexpandedWildcardClasspath = new StringBuilder();
     // Append all entries
     List<String> classPathEntryList = new ArrayList<String>(
       classPathEntries.length);
@@ -1243,16 +1245,22 @@ public class FileUtil {
         continue;
       }
       if (classPathEntry.endsWith("*")) {
+        boolean foundWildCardJar = false;
         // Append all jars that match the wildcard
         Path globPath = new Path(classPathEntry).suffix("{.jar,.JAR}");
         FileStatus[] wildcardJars = FileContext.getLocalFSFileContext().util()
           .globStatus(globPath);
         if (wildcardJars != null) {
           for (FileStatus wildcardJar: wildcardJars) {
+            foundWildCardJar = true;
             classPathEntryList.add(wildcardJar.getPath().toUri().toURL()
               .toExternalForm());
           }
         }
+        if (!foundWildCardJar) {
+          unexpandedWildcardClasspath.append(File.pathSeparator);
+          unexpandedWildcardClasspath.append(classPathEntry);
+        }
       } else {
         // Append just this entry
         File fileCpEntry = null;
@@ -1300,7 +1308,8 @@ public class FileUtil {
     } finally {
       IOUtils.cleanup(LOG, jos, bos, fos);
     }
-
-    return classPathJar.getCanonicalPath();
+    String[] jarCp = {classPathJar.getCanonicalPath(),
+                        unexpandedWildcardClasspath.toString()};
+    return jarCp;
   }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0661342/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Classpath.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Classpath.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Classpath.java
index 0d3df2d..29ffbe1 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Classpath.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Classpath.java
@@ -94,7 +94,7 @@ public final class Classpath {
       final String tmpJarPath;
       try {
         tmpJarPath = FileUtil.createJarWithClassPath(classPath, workingDir,
-          System.getenv());
+          System.getenv())[0];
       } catch (IOException e) {
         terminate(1, "I/O error creating jar: " + e.getMessage());
         return;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0661342/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java
index 2a03988..41794b8 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java
@@ -1036,8 +1036,10 @@ public class TestFileUtil {
     List<String> classPaths = Arrays.asList("", "cp1.jar", "cp2.jar", wildcardPath,
       "cp3.jar", nonExistentSubdir);
     String inputClassPath = StringUtils.join(File.pathSeparator, classPaths);
-    String classPathJar = FileUtil.createJarWithClassPath(inputClassPath,
+    String[] jarCp = FileUtil.createJarWithClassPath(inputClassPath + File.pathSeparator
+ "unexpandedwildcard/*",
       new Path(tmp.getCanonicalPath()), System.getenv());
+    String classPathJar = jarCp[0];
+    assertNotEquals("Unexpanded wildcard was not placed in extra classpath", jarCp[1].indexOf("unexpanded"),
-1);
 
     // verify classpath by reading manifest from jar file
     JarFile jarFile = null;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0661342/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 34d275d..efab925 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -678,6 +678,9 @@ Release 2.6.0 - UNRELEASED
     YARN-2717. Avoided duplicate logging when container logs are not found. (Xuan
     Gong via zjshen)
 
+    YARN-2720. Windows: Wildcard classpath variables not expanded against
+    resources contained in archives. (Craig Welch via cnauroth)
+
 Release 2.5.1 - 2014-09-05
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0661342/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java
index fc6c64f..578a9a2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java
@@ -148,10 +148,12 @@ public class WindowsSecureContainerExecutor extends DefaultContainerExecutor
{
      // Passing CLASSPATH explicitly is *way* too long for command line.
      String classPath = System.getProperty("java.class.path");
      Map<String, String> env = new HashMap<String, String>(System.getenv());
-     String classPathJar = FileUtil.createJarWithClassPath(classPath, appStorageDir, env);
+     String[] jarCp = FileUtil.createJarWithClassPath(classPath, appStorageDir, env);
+     String classPathJar = jarCp[0];
      localizeClasspathJar(new Path(classPathJar), user);
+     String replacementClassPath = classPathJar + jarCp[1];
      command.add("-classpath");
-     command.add(classPathJar);
+     command.add(replacementClassPath);
      
      String javaLibPath = System.getProperty("java.library.path");
      if (javaLibPath != null) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0661342/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
index f87ed6a..30abef5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
@@ -764,11 +764,13 @@ public class ContainerLaunch implements Callable<Integer> {
           System.getenv());
         mergedEnv.putAll(environment);
 
-        String classPathJar = FileUtil.createJarWithClassPath(
+        String[] jarCp = FileUtil.createJarWithClassPath(
           newClassPath.toString(), pwd, mergedEnv);
+        String classPathJar = jarCp[0];
         // In a secure cluster the classpath jar must be localized to grant access
         this.exec.localizeClasspathJar(new Path(classPathJar), container.getUser());
-        environment.put(Environment.CLASSPATH.name(), classPathJar);
+        String replacementClassPath = classPathJar + jarCp[1];
+        environment.put(Environment.CLASSPATH.name(), replacementClassPath);
       }
     }
     // put AuxiliaryService data to environment


Mime
View raw message