tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jeag...@apache.org
Subject tez git commit: TEZ-3537. ArrayIndexOutOfBoundsException with empty environment variables/Port YARN-3768 to Tez (Eric Badger via jeagles)
Date Wed, 07 Dec 2016 22:59:18 GMT
Repository: tez
Updated Branches:
  refs/heads/master a33d2213b -> a2b6eac1c


TEZ-3537. ArrayIndexOutOfBoundsException with empty environment variables/Port YARN-3768 to
Tez (Eric Badger via jeagles)


Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/a2b6eac1
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/a2b6eac1
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/a2b6eac1

Branch: refs/heads/master
Commit: a2b6eac1cc4483126afb96e84c355f253dcfbd85
Parents: a33d221
Author: Jonathan Eagles <jeagles@yahoo-inc.com>
Authored: Wed Dec 7 16:59:04 2016 -0600
Committer: Jonathan Eagles <jeagles@yahoo-inc.com>
Committed: Wed Dec 7 16:59:04 2016 -0600

----------------------------------------------------------------------
 CHANGES.txt                                     |  3 ++
 .../org/apache/tez/common/TezYARNUtils.java     | 25 ++++++++-----
 .../org/apache/tez/common/TestTezYARNUtils.java | 37 ++++++++++++++++++++
 3 files changed, 57 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tez/blob/a2b6eac1/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 6bbbfb2..b15686e 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -7,6 +7,7 @@ INCOMPATIBLE CHANGES
 
 ALL CHANGES:
 
+  TEZ-3537. ArrayIndexOutOfBoundsException with empty environment variables/Port YARN-3768
to Tez
   TEZ-3271. Provide mapreduce failures.maxpercent equivalent.
   TEZ-3222. Reduce messaging overhead for auto-reduce parallelism case.
   TEZ-3547. Add TaskAssignment Analyzer.
@@ -154,6 +155,7 @@ INCOMPATIBLE CHANGES
 
 ALL CHANGES:
 
+  TEZ-3537. ArrayIndexOutOfBoundsException with empty environment variables/Port YARN-3768
to Tez
   TEZ-3536. NPE in WebUIService start when host resolution fails.
   TEZ-3534. Differentiate thread names on Fetchers, minor changes to shuffle shutdown code.
   TEZ-3491. Tez job can hang due to container priority inversion.
@@ -662,6 +664,7 @@ INCOMPATIBLE CHANGES
 
 ALL CHANGES:
 
+  TEZ-3537. ArrayIndexOutOfBoundsException with empty environment variables/Port YARN-3768
to Tez
   TEZ-3536. NPE in WebUIService start when host resolution fails.
   TEZ-3493. DAG submit timeout cannot be set to a month
   TEZ-3505. Move license to the file header for TezBytesWritableSerialization

http://git-wip-us.apache.org/repos/asf/tez/blob/a2b6eac1/tez-api/src/main/java/org/apache/tez/common/TezYARNUtils.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/common/TezYARNUtils.java b/tez-api/src/main/java/org/apache/tez/common/TezYARNUtils.java
index 145f928..d5da672 100644
--- a/tez-api/src/main/java/org/apache/tez/common/TezYARNUtils.java
+++ b/tez-api/src/main/java/org/apache/tez/common/TezYARNUtils.java
@@ -38,8 +38,17 @@ import org.apache.tez.dag.api.TezConstants;
 @Private
 public class TezYARNUtils {
   private static Logger LOG = LoggerFactory.getLogger(TezYARNUtils.class);
-  
-  private static Pattern ENV_VARIABLE_PATTERN = Pattern.compile(Shell.getEnvironmentVariableRegex());
+
+  public static final String ENV_NAME_REGEX = "[A-Za-z_][A-Za-z0-9_]*";
+
+  private static final Pattern VAR_SUBBER =
+    Pattern.compile(Shell.getEnvironmentVariableRegex());
+  private static final Pattern VARVAL_SPLITTER = Pattern.compile(
+    "(?<=^|,)"                            // preceded by ',' or line begin
+      + '(' + ENV_NAME_REGEX + ')'      // var group
+      + '='
+      + "([^,]*)"                             // val group
+  );
 
   public static String getFrameworkClasspath(Configuration conf, boolean usingArchive) {
     StringBuilder classpathBuilder = new StringBuilder();
@@ -126,10 +135,10 @@ public class TezYARNUtils {
   public static void appendToEnvFromInputString(Map<String, String> env,
       String envString, String classPathSeparator) {
     if (envString != null && envString.length() > 0) {
-      String childEnvs[] = envString.split(",");
-      for (String cEnv : childEnvs) {
-        String[] parts = cEnv.split("="); // split on '='
-        Matcher m = ENV_VARIABLE_PATTERN.matcher(parts[1]);
+      Matcher varValMatcher = VARVAL_SPLITTER.matcher(envString);
+      while (varValMatcher.find()) {
+        String envVar = varValMatcher.group(1);
+        Matcher m = VAR_SUBBER.matcher(varValMatcher.group(2));
         StringBuffer sb = new StringBuffer();
         while (m.find()) {
           String var = m.group(1);
@@ -145,7 +154,7 @@ public class TezYARNUtils {
             m.appendReplacement(sb, Matcher.quoteReplacement(replace));
         }
         m.appendTail(sb);
-        addToEnvironment(env, parts[0], sb.toString(), classPathSeparator);
+        addToEnvironment(env, envVar, sb.toString(), classPathSeparator);
       }
     }
   }
@@ -156,7 +165,7 @@ public class TezYARNUtils {
       String childEnvs[] = envString.split(",");
       for (String cEnv : childEnvs) {
         String[] parts = cEnv.split("="); // split on '='
-        Matcher m = ENV_VARIABLE_PATTERN.matcher(parts[1]);
+        Matcher m = VAR_SUBBER .matcher(parts[1]);
         StringBuffer sb = new StringBuffer();
         while (m.find()) {
           String var = m.group(1);

http://git-wip-us.apache.org/repos/asf/tez/blob/a2b6eac1/tez-api/src/test/java/org/apache/tez/common/TestTezYARNUtils.java
----------------------------------------------------------------------
diff --git a/tez-api/src/test/java/org/apache/tez/common/TestTezYARNUtils.java b/tez-api/src/test/java/org/apache/tez/common/TestTezYARNUtils.java
index 092bd50..9f02801 100644
--- a/tez-api/src/test/java/org/apache/tez/common/TestTezYARNUtils.java
+++ b/tez-api/src/test/java/org/apache/tez/common/TestTezYARNUtils.java
@@ -19,16 +19,21 @@
 package org.apache.tez.common;
 
 import java.io.File;
+import java.util.HashMap;
 import java.util.Map;
 import java.util.TreeMap;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.util.Shell;
 import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
 import org.apache.tez.dag.api.TezConfiguration;
 import org.apache.tez.dag.api.TezConstants;
 import org.junit.Assert;
 import org.junit.Test;
 
+import static org.apache.tez.common.TezYARNUtils.appendToEnvFromInputString;
+import static org.junit.Assert.assertEquals;
+
 public class TestTezYARNUtils {
 
   @Test(timeout = 5000)
@@ -104,4 +109,36 @@ public class TestTezYARNUtils {
     Assert.assertTrue(classpath.indexOf("foobar") >
         classpath.indexOf(Environment.PWD.$()));
   }
+
+  @Test
+  public void testSetEnvFromInputString() {
+    Map<String, String> environment = new HashMap<String, String>();
+    environment.put("JAVA_HOME", "/path/jdk");
+    String goodEnv = "a1=1,b_2=2,_c=3,d=4,e=,f_win=%JAVA_HOME%"
+      + ",g_nix=$JAVA_HOME";
+    appendToEnvFromInputString(environment, goodEnv, File.pathSeparator);
+    assertEquals("1", environment.get("a1"));
+    assertEquals("2", environment.get("b_2"));
+    assertEquals("3", environment.get("_c"));
+    assertEquals("4", environment.get("d"));
+    assertEquals("", environment.get("e"));
+    if (Shell.WINDOWS) {
+      assertEquals("$JAVA_HOME", environment.get("g_nix"));
+      assertEquals("/path/jdk", environment.get("f_win"));
+    } else {
+      assertEquals("/path/jdk", environment.get("g_nix"));
+      assertEquals("%JAVA_HOME%", environment.get("f_win"));
+    }
+    String badEnv = "0=,1,,2=a=b,3=a=,4==,5==a,==,c-3=3,=";
+    environment.clear();
+    appendToEnvFromInputString(environment, badEnv, File.pathSeparator);
+    assertEquals(environment.size(), 0);
+
+    // Test "=" in the value part
+    environment.clear();
+    appendToEnvFromInputString(environment, "b1,e1==,e2=a1=a2,b2",
+      File.pathSeparator);
+    assertEquals("=", environment.get("e1"));
+    assertEquals("a1=a2", environment.get("e2"));
+  }
 }


Mime
View raw message