tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ss...@apache.org
Subject git commit: TEZ-939. Fix build break caused by changes in YARN-1824. (sseth)
Date Tue, 18 Mar 2014 17:44:21 GMT
Repository: incubator-tez
Updated Branches:
  refs/heads/master ae6c9746c -> 4a60f05c1


TEZ-939. Fix build break caused by changes in YARN-1824. (sseth)


Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/4a60f05c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/4a60f05c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/4a60f05c

Branch: refs/heads/master
Commit: 4a60f05c105188aefb48f64c4ccf2efbc4b26da0
Parents: ae6c974
Author: Siddharth Seth <sseth@apache.org>
Authored: Tue Mar 18 10:43:30 2014 -0700
Committer: Siddharth Seth <sseth@apache.org>
Committed: Tue Mar 18 10:43:30 2014 -0700

----------------------------------------------------------------------
 .../org/apache/tez/client/TezClientUtils.java   | 20 +++---
 .../java/org/apache/tez/client/TezSession.java  | 11 +--
 .../org/apache/tez/common/TezYARNUtils.java     | 72 ++++++++++++++++++++
 .../apache/tez/mapreduce/hadoop/MRHelpers.java  | 26 +++----
 4 files changed, 102 insertions(+), 27 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/4a60f05c/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
index ff45cf5..d47ff80 100644
--- a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
+++ b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
@@ -70,10 +70,10 @@ import org.apache.hadoop.yarn.client.api.YarnClient;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.security.client.ClientToAMTokenIdentifier;
-import org.apache.hadoop.yarn.util.Apps;
 import org.apache.hadoop.yarn.util.ConverterUtils;
 import org.apache.hadoop.yarn.util.Records;
 import org.apache.log4j.Level;
+import org.apache.tez.common.TezYARNUtils;
 import org.apache.tez.common.impl.LogUtils;
 import org.apache.tez.common.security.TokenCache;
 import org.apache.tez.dag.api.DAG;
@@ -401,7 +401,8 @@ public class TezClientUtils {
 
     if (amConfig.getEnv() != null) {
       for (Map.Entry<String, String> entry : amConfig.getEnv().entrySet()) {
-        Apps.addToEnvironment(environment, entry.getKey(), entry.getValue());
+        TezYARNUtils.addToEnvironment(environment, entry.getKey(), entry.getValue(),
+            File.pathSeparator);
       }
     }
 
@@ -567,20 +568,22 @@ public class TezClientUtils {
   static String getFrameworkClasspath(Configuration conf) {
     Map<String, String> environment = new HashMap<String, String>();
 
-    Apps.addToEnvironment(environment,
+    TezYARNUtils.addToEnvironment(environment,
         Environment.CLASSPATH.name(),
-        Environment.PWD.$());
+        Environment.PWD.$(),
+        File.pathSeparator);
 
-    Apps.addToEnvironment(environment,
+    TezYARNUtils.addToEnvironment(environment,
         Environment.CLASSPATH.name(),
-        Environment.PWD.$() + File.separator + "*");
+        Environment.PWD.$() + File.separator + "*",
+        File.pathSeparator);
 
     // Add YARN/COMMON/HDFS jars and conf locations to path
     for (String c : conf.getStrings(
         YarnConfiguration.YARN_APPLICATION_CLASSPATH,
         YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) {
-      Apps.addToEnvironment(environment, Environment.CLASSPATH.name(),
-          c.trim());
+      TezYARNUtils.addToEnvironment(environment, Environment.CLASSPATH.name(),
+          c.trim(), File.pathSeparator);
     }
     return environment.get(Environment.CLASSPATH.name());
   }
@@ -749,5 +752,4 @@ public class TezClientUtils {
     }
     return proxy;
   }
-
 }

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/4a60f05c/tez-api/src/main/java/org/apache/tez/client/TezSession.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/client/TezSession.java b/tez-api/src/main/java/org/apache/tez/client/TezSession.java
index 6ea80a9..f16e509 100644
--- a/tez-api/src/main/java/org/apache/tez/client/TezSession.java
+++ b/tez-api/src/main/java/org/apache/tez/client/TezSession.java
@@ -18,6 +18,7 @@
 
 package org.apache.tez.client;
 
+import java.io.File;
 import java.io.IOException;
 import java.util.Map;
 
@@ -33,7 +34,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
 import org.apache.hadoop.yarn.api.records.LocalResource;
 import org.apache.hadoop.yarn.client.api.YarnClient;
 import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.util.Apps;
+import org.apache.tez.common.TezYARNUtils;
 import org.apache.tez.dag.api.DAG;
 import org.apache.tez.dag.api.DAGSubmissionTimedOut;
 import org.apache.tez.dag.api.DagTypeConverters;
@@ -150,9 +151,9 @@ public class TezSession {
         .getFrameworkClasspath(sessionConfig.getYarnConfiguration());
     for (Vertex v : dag.getVertices()) {
       Map<String, String> taskEnv = v.getTaskEnvironment();
-      Apps.addToEnvironment(taskEnv,
+      TezYARNUtils.addToEnvironment(taskEnv,
           ApplicationConstants.Environment.CLASSPATH.name(),
-          classpath);
+          classpath, File.pathSeparator);
     }
     
     DAGPlan dagPlan = dag.createDag(sessionConfig.getTezConfiguration());
@@ -295,9 +296,9 @@ public class TezSession {
       String classpath = TezClientUtils
         .getFrameworkClasspath(sessionConfig.getYarnConfiguration());
       Map<String, String> contextEnv = context.getEnvironment();
-      Apps.addToEnvironment(contextEnv,
+      TezYARNUtils.addToEnvironment(contextEnv,
         ApplicationConstants.Environment.CLASSPATH.name(),
-        classpath);
+        classpath, File.pathSeparator);
 
       DAGClientAMProtocolRPC.PreWarmRequestProto.Builder
         preWarmReqProtoBuilder =

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/4a60f05c/tez-api/src/main/java/org/apache/tez/common/TezYARNUtils.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/common/TezYARNUtils.java b/tez-api/src/main/java/org/apache/tez/common/TezYARNUtils.java
new file mode 100644
index 0000000..aceec81
--- /dev/null
+++ b/tez-api/src/main/java/org/apache/tez/common/TezYARNUtils.java
@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.tez.common;
+
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.util.Shell;
+import org.apache.hadoop.util.StringInterner;
+
+@Private
+public class TezYARNUtils {
+
+  public static void setEnvFromInputString(Map<String, String> env,
+      String envString,  String classPathSeparator) {
+    if (envString != null && envString.length() > 0) {
+      String childEnvs[] = envString.split(",");
+      Pattern p = Pattern.compile(Shell.getEnvironmentVariableRegex());
+      for (String cEnv : childEnvs) {
+        String[] parts = cEnv.split("="); // split on '='
+        Matcher m = p.matcher(parts[1]);
+        StringBuffer sb = new StringBuffer();
+        while (m.find()) {
+          String var = m.group(1);
+          // replace $env with the child's env constructed by tt's
+          String replace = env.get(var);
+          // if this key is not configured by the tt for the child .. get it
+          // from the tt's env
+          if (replace == null)
+            replace = System.getenv(var);
+          // the env key is note present anywhere .. simply set it
+          if (replace == null)
+            replace = "";
+          m.appendReplacement(sb, Matcher.quoteReplacement(replace));
+        }
+        m.appendTail(sb);
+        addToEnvironment(env, parts[0], sb.toString(), classPathSeparator);
+      }
+    }
+  }
+  
+  public static void addToEnvironment(
+      Map<String, String> environment,
+      String variable, String value, String classPathSeparator) {
+    String val = environment.get(variable);
+    if (val == null) {
+      val = value;
+    } else {
+      val = val + classPathSeparator + value;
+    }
+    environment.put(StringInterner.weakIntern(variable), 
+        StringInterner.weakIntern(val));
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/4a60f05c/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRHelpers.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRHelpers.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRHelpers.java
index 91fa4ae..3ed3702 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRHelpers.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRHelpers.java
@@ -19,6 +19,7 @@
 package org.apache.tez.mapreduce.hadoop;
 
 import java.io.DataOutputStream;
+import java.io.File;
 import java.io.IOException;
 import java.net.InetAddress;
 import java.util.ArrayList;
@@ -49,7 +50,6 @@ import org.apache.hadoop.mapreduce.JobContext;
 import org.apache.hadoop.mapreduce.JobSubmissionFiles;
 import org.apache.hadoop.mapreduce.split.JobSplitWriter;
 import org.apache.hadoop.mapreduce.v2.proto.MRProtos;
-import org.apache.hadoop.mapreduce.v2.util.MRApps;
 import org.apache.hadoop.util.ReflectionUtils;
 import org.apache.hadoop.yarn.ContainerLogAppender;
 import org.apache.hadoop.yarn.api.ApplicationConstants;
@@ -59,10 +59,10 @@ import org.apache.hadoop.yarn.api.records.LocalResourceType;
 import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.util.Apps;
 import org.apache.hadoop.yarn.util.ConverterUtils;
 import org.apache.tez.common.TezJobConfig;
 import org.apache.tez.common.TezUtils;
+import org.apache.tez.common.TezYARNUtils;
 import org.apache.tez.dag.api.InputDescriptor;
 import org.apache.tez.dag.api.OutputDescriptor;
 import org.apache.tez.dag.api.TezConfiguration;
@@ -819,19 +819,20 @@ public class MRHelpers {
         MRJobConfig.MAPRED_ADMIN_USER_SHELL, MRJobConfig.DEFAULT_SHELL));
 
     // Add pwd to LD_LIBRARY_PATH, add this before adding anything else
-    Apps.addToEnvironment(environment, Environment.LD_LIBRARY_PATH.name(),
-        Environment.PWD.$());
+    TezYARNUtils.addToEnvironment(environment, Environment.LD_LIBRARY_PATH.name(),
+        Environment.PWD.$(), File.pathSeparator);
 
     // Add the env variables passed by the admin
-    Apps.setEnvFromInputString(environment, conf.get(
+    TezYARNUtils.setEnvFromInputString(environment, conf.get(
         MRJobConfig.MAPRED_ADMIN_USER_ENV,
-        MRJobConfig.DEFAULT_MAPRED_ADMIN_USER_ENV));
+        MRJobConfig.DEFAULT_MAPRED_ADMIN_USER_ENV),
+        File.pathSeparator);
 
     // Add the env variables passed by the user
     String mapredChildEnv = (isMap ?
         conf.get(MRJobConfig.MAP_ENV, "")
         : conf.get(MRJobConfig.REDUCE_ENV, ""));
-    Apps.setEnvFromInputString(environment, mapredChildEnv);
+    TezYARNUtils.setEnvFromInputString(environment, mapredChildEnv, File.pathSeparator);
 
     // Set logging level in the environment.
     environment.put(
@@ -869,12 +870,11 @@ public class MRHelpers {
    * @param conf Configuration from which to extract information
    * @param environment Environment map to update
    */
-  public static void updateEnvironmentForMRAM(Configuration conf,
-      Map<String, String> environment) {
-    MRApps.setEnvFromInputString(environment,
-        conf.get(MRJobConfig.MR_AM_ADMIN_USER_ENV));
-    MRApps.setEnvFromInputString(environment,
-        conf.get(MRJobConfig.MR_AM_ENV));
+  public static void updateEnvironmentForMRAM(Configuration conf, Map<String, String>
environment) {
+    TezYARNUtils.setEnvFromInputString(environment, conf.get(MRJobConfig.MR_AM_ADMIN_USER_ENV),
+        File.pathSeparator);
+    TezYARNUtils.setEnvFromInputString(environment, conf.get(MRJobConfig.MR_AM_ENV),
+        File.pathSeparator);
   }
 
   /**


Mime
View raw message