tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ss...@apache.org
Subject [04/37] tez git commit: TEZ-2364. Resolve config parameter replacement on the client, before sending them to the AM. (sseth)
Date Tue, 28 Apr 2015 20:40:45 GMT
TEZ-2364. Resolve config parameter replacement on the client, before
sending them to the AM. (sseth)


Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/2935ef41
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/2935ef41
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/2935ef41

Branch: refs/heads/TEZ-2003
Commit: 2935ef415642a9869166a1d5e1484a635e4401a2
Parents: bbbebb8
Author: Siddharth Seth <sseth@apache.org>
Authored: Fri Apr 24 13:42:01 2015 -0700
Committer: Siddharth Seth <sseth@apache.org>
Committed: Fri Apr 24 13:42:01 2015 -0700

----------------------------------------------------------------------
 CHANGES.txt                                     |  1 +
 .../org/apache/tez/client/TezClientUtils.java   |  2 +-
 .../apache/tez/dag/api/TezConfiguration.java    |  2 +-
 .../apache/tez/client/TestTezClientUtils.java   | 36 ++++++++++++++++----
 4 files changed, 33 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tez/blob/2935ef41/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index e21d178..92c1f6e 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -9,6 +9,7 @@ INCOMPATIBLE CHANGES
   TEZ-1993. Implement a pluggable InputSizeEstimator for grouping fairly
 
 ALL CHANGES:
+  TEZ-2364. Resolve config parameter replacement on the client, before sending them to the
AM.
   TEZ-2298. Avoid logging full exception trace in TaskRunner when it's not the main error
reason and is ignored.
   TEZ-2248. VertexImpl/DAGImpl.checkForCompletion have too many termination cause checks
   TEZ-2341. TestMockDAGAppMaster.testBasicCounters fails on windows

http://git-wip-us.apache.org/repos/asf/tez/blob/2935ef41/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
index d12ece2..86ba0f9 100644
--- a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
+++ b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
@@ -743,7 +743,7 @@ public class TezClientUtils {
     for (Entry<String, String> entry : amConf) {
       PlanKeyValuePair.Builder kvp = PlanKeyValuePair.newBuilder();
       kvp.setKey(entry.getKey());
-      kvp.setValue(entry.getValue());
+      kvp.setValue(amConf.get(entry.getKey()));
       builder.addConfKeyValues(kvp);
     }
     if (additionalConfigs != null && !additionalConfigs.isEmpty()) {

http://git-wip-us.apache.org/repos/asf/tez/blob/2935ef41/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
index 7a49ee5..50b3e00 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
@@ -133,7 +133,7 @@ public class TezConfiguration extends Configuration {
   public static final String TEZ_AM_STAGING_DIR = TEZ_PREFIX + "staging-dir";
   public static final String TEZ_AM_STAGING_DIR_DEFAULT = "/tmp/"
       + System.getProperty("user.name") + "/tez/staging";
-  
+
   /**
    * String value that is a file path.
    * Path to a credentials file (with serialized credentials) located on the local file system.

http://git-wip-us.apache.org/repos/asf/tez/blob/2935ef41/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java
----------------------------------------------------------------------
diff --git a/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java b/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java
index ea73ab3..cfba917 100644
--- a/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java
+++ b/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java
@@ -46,7 +46,6 @@ import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.hdfs.DistributedFileSystem;
 import org.apache.hadoop.io.DataInputByteBuffer;
-import org.apache.hadoop.io.Text;
 import org.apache.hadoop.security.Credentials;
 import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.yarn.api.ApplicationConstants;
@@ -68,7 +67,8 @@ import org.apache.tez.dag.api.TezConfiguration;
 import org.apache.tez.dag.api.TezConstants;
 import org.apache.tez.dag.api.TezUncheckedException;
 import org.apache.tez.dag.api.Vertex;
-import org.apache.tez.dag.api.records.DAGProtos;
+import org.apache.tez.dag.api.records.DAGProtos.ConfigurationProto;
+import org.apache.tez.dag.api.records.DAGProtos.PlanKeyValuePair;
 import org.junit.Assert;
 import org.junit.Test;
 
@@ -283,7 +283,8 @@ public class TestTezClientUtils {
   public void testAMLoggingOptsPerLogger() throws IOException, YarnException {
 
     TezConfiguration tezConf = new TezConfiguration();
-    tezConf.set(TezConfiguration.TEZ_AM_LOG_LEVEL, "WARN;org.apache.hadoop.ipc=DEBUG;org.apache.hadoop.security=DEBUG");
+    tezConf.set(TezConfiguration.TEZ_AM_LOG_LEVEL,
+        "WARN;org.apache.hadoop.ipc=DEBUG;org.apache.hadoop.security=DEBUG");
 
     ApplicationId appId = ApplicationId.newInstance(1000, 1);
     Credentials credentials = new Credentials();
@@ -301,7 +302,8 @@ public class TestTezClientUtils {
 
     List<String> expectedCommands = new LinkedList<String>();
     expectedCommands.add("-Dlog4j.configuratorClass=org.apache.tez.common.TezLog4jConfigurator");
-    expectedCommands.add("-Dlog4j.configuration=" + TezConstants.TEZ_CONTAINER_LOG4J_PROPERTIES_FILE);
+    expectedCommands.add(
+        "-Dlog4j.configuration=" + TezConstants.TEZ_CONTAINER_LOG4J_PROPERTIES_FILE);
     expectedCommands.add("-D" + YarnConfiguration.YARN_APP_CONTAINER_LOG_DIR + "=" +
         ApplicationConstants.LOG_DIR_EXPANSION_VAR);
     expectedCommands.add("-D" + TezConstants.TEZ_ROOT_LOGGER_NAME + "=" + "WARN" + "," +
@@ -465,6 +467,28 @@ public class TestTezClientUtils {
         && javaOpts.contains("-Dlog4j.configuratorClass=org.apache.tez.common.TezLog4jConfigurator"));
   }
 
+  @Test (timeout = 5000)
+  public void testConfSerializationForAm() {
+    Configuration conf =new Configuration(false);
+    String val1 = "fixedProperty";
+    String val2 = "parametrizedProperty/${user.name}";
+    String expVal2 = "parametrizedProperty/" + System.getProperty("user.name");
+    conf.set("property1", val1);
+    conf.set("property2", val2);
+
+    Map<String, String> expected = new HashMap<String, String>();
+    expected.put("property1", val1);
+    expected.put("property2", expVal2);
+
+    ConfigurationProto confProto = TezClientUtils.createFinalConfProtoForApp(conf, null);
+
+    for (PlanKeyValuePair kvPair : confProto.getConfKeyValuesList()) {
+      String v = expected.remove(kvPair.getKey());
+      assertEquals(v, kvPair.getValue());
+    }
+    assertTrue(expected.isEmpty());
+  }
+
   // To run this test case see TestTezCommonUtils::testLocalResourceVisibility
   // We do not have much control over the directory structure, cannot mock as the functions
are
   // static and do not want to spin up a minitez cluster just for this.
@@ -560,9 +584,9 @@ public class TestTezClientUtils {
       srcConf.set(entry.getKey(), entry.getValue());
     }
 
-    DAGProtos.ConfigurationProto confProto = TezClientUtils.createFinalConfProtoForApp(srcConf);
+    ConfigurationProto confProto = TezClientUtils.createFinalConfProtoForApp(srcConf);
 
-    for (DAGProtos.PlanKeyValuePair kvPair : confProto.getConfKeyValuesList()) {
+    for (PlanKeyValuePair kvPair : confProto.getConfKeyValuesList()) {
       String val = confMap.remove(kvPair.getKey());
       assertNotNull(val);
       assertEquals(val, kvPair.getValue());


Mime
View raw message