hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From aengin...@apache.org
Subject [01/17] hadoop git commit: YARN-4958. The file localization process should allow for wildcards to reduce the application footprint in the state store (Daniel Templeton via sjlee)
Date Wed, 22 Jun 2016 00:39:00 GMT
Repository: hadoop
Updated Branches:
  refs/heads/HDFS-1312 90a032653 -> 62e4dcd2d


YARN-4958. The file localization process should allow for wildcards to reduce the application
footprint in the state store (Daniel Templeton via sjlee)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5107a967
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5107a967
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5107a967

Branch: refs/heads/HDFS-1312
Commit: 5107a967fa2558deba11c33a326d4d2e5748f452
Parents: fc6b50c
Author: Sangjin Lee <sjlee@twitter.com>
Authored: Mon Jun 20 09:56:53 2016 -0700
Committer: Sangjin Lee <sjlee@twitter.com>
Committed: Mon Jun 20 09:56:53 2016 -0700

----------------------------------------------------------------------
 .../yarn/server/nodemanager/ContainerExecutor.java   | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/5107a967/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
index 98171af..98d45f4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
@@ -61,7 +61,7 @@ import org.apache.hadoop.util.Shell;
 import org.apache.hadoop.util.StringUtils;
 
 public abstract class ContainerExecutor implements Configurable {
-
+  private static final String WILDCARD = "*";
   private static final Log LOG = LogFactory.getLog(ContainerExecutor.class);
   final public static FsPermission TASK_LAUNCH_SCRIPT_PERMISSION =
     FsPermission.createImmutable((short) 0700);
@@ -281,7 +281,18 @@ public abstract class ContainerExecutor implements Configurable {
     if (resources != null) {
       for (Map.Entry<Path,List<String>> entry : resources.entrySet()) {
         for (String linkName : entry.getValue()) {
-          sb.symlink(entry.getKey(), new Path(linkName));
+          if (new Path(linkName).getName().equals(WILDCARD)) {
+            // If this is a wildcarded path, link to everything in the
+            // directory from the working directory
+            File directory = new File(entry.getKey().toString());
+
+            for (File wildLink : directory.listFiles()) {
+              sb.symlink(new Path(wildLink.toString()),
+                  new Path(wildLink.getName()));
+            }
+          } else {
+            sb.symlink(entry.getKey(), new Path(linkName));
+          }
         }
       }
     }


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message