tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jeag...@apache.org
Subject tez git commit: TEZ-3537. ArrayIndexOutOfBoundsException with empty environment variables/Port YARN-3768 to Tez (Eric Badger via jeagles)
Date Wed, 07 Dec 2016 23:23:38 GMT
Repository: tez
Updated Branches:
  refs/heads/branch-0.8 45ed926e5 -> 9d9ee869a


TEZ-3537. ArrayIndexOutOfBoundsException with empty environment variables/Port YARN-3768 to
Tez (Eric Badger via jeagles)

(cherry picked from commit a2b6eac1cc4483126afb96e84c355f253dcfbd85)


Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/9d9ee869
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/9d9ee869
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/9d9ee869

Branch: refs/heads/branch-0.8
Commit: 9d9ee869aec2def27bdf632ebe2f7f8b783c98b5
Parents: 45ed926
Author: Jonathan Eagles <jeagles@yahoo-inc.com>
Authored: Wed Dec 7 16:59:04 2016 -0600
Committer: Jonathan Eagles <jeagles@yahoo-inc.com>
Committed: Wed Dec 7 17:10:40 2016 -0600

----------------------------------------------------------------------
 CHANGES.txt                                     |  2 ++
 .../org/apache/tez/common/TezYARNUtils.java     | 25 ++++++++-----
 .../org/apache/tez/common/TestTezYARNUtils.java | 37 ++++++++++++++++++++
 3 files changed, 56 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tez/blob/9d9ee869/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 17e3958..06a0a11 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -7,6 +7,7 @@ INCOMPATIBLE CHANGES
 
 ALL CHANGES:
 
+  TEZ-3537. ArrayIndexOutOfBoundsException with empty environment variables/Port YARN-3768
to Tez
   TEZ-3507. Task logs link when editing url from one task to another.
   TEZ-3536. NPE in WebUIService start when host resolution fails.
   TEZ-3534. Differentiate thread names on Fetchers, minor changes to shuffle shutdown code.
@@ -532,6 +533,7 @@ INCOMPATIBLE CHANGES
 
 ALL CHANGES:
 
+  TEZ-3537. ArrayIndexOutOfBoundsException with empty environment variables/Port YARN-3768
to Tez
   TEZ-3507. Task logs link when editing url from one task to another.
   TEZ-3536. NPE in WebUIService start when host resolution fails.
   TEZ-3493. DAG submit timeout cannot be set to a month

http://git-wip-us.apache.org/repos/asf/tez/blob/9d9ee869/tez-api/src/main/java/org/apache/tez/common/TezYARNUtils.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/common/TezYARNUtils.java b/tez-api/src/main/java/org/apache/tez/common/TezYARNUtils.java
index c505ca8..8eb0766 100644
--- a/tez-api/src/main/java/org/apache/tez/common/TezYARNUtils.java
+++ b/tez-api/src/main/java/org/apache/tez/common/TezYARNUtils.java
@@ -38,8 +38,17 @@ import org.apache.tez.dag.api.TezConstants;
 @Private
 public class TezYARNUtils {
   private static Logger LOG = LoggerFactory.getLogger(TezYARNUtils.class);
-  
-  private static Pattern ENV_VARIABLE_PATTERN = Pattern.compile(Shell.getEnvironmentVariableRegex());
+
+  public static final String ENV_NAME_REGEX = "[A-Za-z_][A-Za-z0-9_]*";
+
+  private static final Pattern VAR_SUBBER =
+    Pattern.compile(Shell.getEnvironmentVariableRegex());
+  private static final Pattern VARVAL_SPLITTER = Pattern.compile(
+    "(?<=^|,)"                            // preceded by ',' or line begin
+      + '(' + ENV_NAME_REGEX + ')'      // var group
+      + '='
+      + "([^,]*)"                             // val group
+  );
 
   public static String getFrameworkClasspath(Configuration conf, boolean usingArchive) {
     StringBuilder classpathBuilder = new StringBuilder();
@@ -114,10 +123,10 @@ public class TezYARNUtils {
   public static void appendToEnvFromInputString(Map<String, String> env,
       String envString, String classPathSeparator) {
     if (envString != null && envString.length() > 0) {
-      String childEnvs[] = envString.split(",");
-      for (String cEnv : childEnvs) {
-        String[] parts = cEnv.split("="); // split on '='
-        Matcher m = ENV_VARIABLE_PATTERN.matcher(parts[1]);
+      Matcher varValMatcher = VARVAL_SPLITTER.matcher(envString);
+      while (varValMatcher.find()) {
+        String envVar = varValMatcher.group(1);
+        Matcher m = VAR_SUBBER.matcher(varValMatcher.group(2));
         StringBuffer sb = new StringBuffer();
         while (m.find()) {
           String var = m.group(1);
@@ -133,7 +142,7 @@ public class TezYARNUtils {
             m.appendReplacement(sb, Matcher.quoteReplacement(replace));
         }
         m.appendTail(sb);
-        addToEnvironment(env, parts[0], sb.toString(), classPathSeparator);
+        addToEnvironment(env, envVar, sb.toString(), classPathSeparator);
       }
     }
   }
@@ -144,7 +153,7 @@ public class TezYARNUtils {
       String childEnvs[] = envString.split(",");
       for (String cEnv : childEnvs) {
         String[] parts = cEnv.split("="); // split on '='
-        Matcher m = ENV_VARIABLE_PATTERN.matcher(parts[1]);
+        Matcher m = VAR_SUBBER .matcher(parts[1]);
         StringBuffer sb = new StringBuffer();
         while (m.find()) {
           String var = m.group(1);

http://git-wip-us.apache.org/repos/asf/tez/blob/9d9ee869/tez-api/src/test/java/org/apache/tez/common/TestTezYARNUtils.java
----------------------------------------------------------------------
diff --git a/tez-api/src/test/java/org/apache/tez/common/TestTezYARNUtils.java b/tez-api/src/test/java/org/apache/tez/common/TestTezYARNUtils.java
index 2dabf51..262e4ba 100644
--- a/tez-api/src/test/java/org/apache/tez/common/TestTezYARNUtils.java
+++ b/tez-api/src/test/java/org/apache/tez/common/TestTezYARNUtils.java
@@ -19,16 +19,21 @@
 package org.apache.tez.common;
 
 import java.io.File;
+import java.util.HashMap;
 import java.util.Map;
 import java.util.TreeMap;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.util.Shell;
 import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
 import org.apache.tez.dag.api.TezConfiguration;
 import org.apache.tez.dag.api.TezConstants;
 import org.junit.Assert;
 import org.junit.Test;
 
+import static org.apache.tez.common.TezYARNUtils.appendToEnvFromInputString;
+import static org.junit.Assert.assertEquals;
+
 public class TestTezYARNUtils {
 
   @Test(timeout = 5000)
@@ -91,4 +96,36 @@ public class TestTezYARNUtils {
     Assert.assertTrue(classpath.indexOf("foobar") >
         classpath.indexOf(Environment.PWD.$()));
   }
+
+  @Test
+  public void testSetEnvFromInputString() {
+    Map<String, String> environment = new HashMap<String, String>();
+    environment.put("JAVA_HOME", "/path/jdk");
+    String goodEnv = "a1=1,b_2=2,_c=3,d=4,e=,f_win=%JAVA_HOME%"
+      + ",g_nix=$JAVA_HOME";
+    appendToEnvFromInputString(environment, goodEnv, File.pathSeparator);
+    assertEquals("1", environment.get("a1"));
+    assertEquals("2", environment.get("b_2"));
+    assertEquals("3", environment.get("_c"));
+    assertEquals("4", environment.get("d"));
+    assertEquals("", environment.get("e"));
+    if (Shell.WINDOWS) {
+      assertEquals("$JAVA_HOME", environment.get("g_nix"));
+      assertEquals("/path/jdk", environment.get("f_win"));
+    } else {
+      assertEquals("/path/jdk", environment.get("g_nix"));
+      assertEquals("%JAVA_HOME%", environment.get("f_win"));
+    }
+    String badEnv = "0=,1,,2=a=b,3=a=,4==,5==a,==,c-3=3,=";
+    environment.clear();
+    appendToEnvFromInputString(environment, badEnv, File.pathSeparator);
+    assertEquals(environment.size(), 0);
+
+    // Test "=" in the value part
+    environment.clear();
+    appendToEnvFromInputString(environment, "b1,e1==,e2=a1=a2,b2",
+      File.pathSeparator);
+    assertEquals("=", environment.get("e1"));
+    assertEquals("a1=a2", environment.get("e2"));
+  }
 }


Mime
View raw message