tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ss...@apache.org
Subject git commit: TEZ-1534. Make client side configs available to AM and tasks. (sseth) (cherry picked from commit 90ef95ad6742fb12de43bcfbb10c3a74af15233c)
Date Thu, 11 Sep 2014 07:20:51 GMT
Repository: tez
Updated Branches:
  refs/heads/branch-0.5 176b12cde -> fba3a5121


TEZ-1534. Make client side configs available to AM and tasks. (sseth)
(cherry picked from commit 90ef95ad6742fb12de43bcfbb10c3a74af15233c)


Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/fba3a512
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/fba3a512
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/fba3a512

Branch: refs/heads/branch-0.5
Commit: fba3a5121d00016fb055f96dbe8acc58d834e134
Parents: 176b12c
Author: Siddharth Seth <sseth@apache.org>
Authored: Thu Sep 11 00:19:41 2014 -0700
Committer: Siddharth Seth <sseth@apache.org>
Committed: Thu Sep 11 00:20:40 2014 -0700

----------------------------------------------------------------------
 CHANGES.txt                                     |  1 +
 .../org/apache/tez/client/TezClientUtils.java   | 48 ++++++--------------
 .../apache/tez/dag/api/TezConfiguration.java    |  1 -
 .../apache/tez/client/TestTezClientUtils.java   | 27 +++++++++++
 4 files changed, 41 insertions(+), 36 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tez/blob/fba3a512/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 7b7accb..f27ebeb 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -12,6 +12,7 @@ ALL CHANGES
   TEZ-1447. Provide a mechanism for InputInitializers to know about Vertex state changes.
   TEZ-1362. Remove DAG_COMPLETED in DAGEventType. 
   TEZ-1519. TezTaskRunner should not initialize TezConfiguration in TezChild.
+  TEZ-1534. Make client side configs available to AM and tasks.
 
 Release 0.5.0: Unreleased
 

http://git-wip-us.apache.org/repos/asf/tez/blob/fba3a512/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
index b53f00c..adda0a9 100644
--- a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
+++ b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
@@ -375,7 +375,6 @@ public class TezClientUtils {
 
   /**
    * Create an ApplicationSubmissionContext to launch a Tez AM
-   * @param conf TezConfiguration
    * @param appId Application Id
    * @param dag DAG to be submitted
    * @param amName Name for the application
@@ -501,23 +500,12 @@ public class TezClientUtils {
     amLocalResources.putAll(tezJarResources);
 
     // emit conf as PB file
-    Configuration finalTezConf = createFinalTezConfForApp(amConfig.getTezConfiguration());
+    ConfigurationProto finalConfProto = createFinalConfProtoForApp(amConfig.getTezConfiguration());
     
     FSDataOutputStream amConfPBOutBinaryStream = null;
     try {
-      ConfigurationProto.Builder confProtoBuilder =
-          ConfigurationProto.newBuilder();
-      Iterator<Entry<String, String>> iter = finalTezConf.iterator();
-      while (iter.hasNext()) {
-        Entry<String, String> entry = iter.next();
-        PlanKeyValuePair.Builder kvp = PlanKeyValuePair.newBuilder();
-        kvp.setKey(entry.getKey());
-        kvp.setValue(entry.getValue());
-        confProtoBuilder.addConfKeyValues(kvp);
-      }
-      //binary output
       amConfPBOutBinaryStream = TezCommonUtils.createFileForAM(fs, binaryConfPath);
-      confProtoBuilder.build().writeTo(amConfPBOutBinaryStream);
+      finalConfProto.writeTo(amConfPBOutBinaryStream);
     } finally {
       if(amConfPBOutBinaryStream != null){
         amConfPBOutBinaryStream.close();
@@ -560,7 +548,7 @@ public class TezClientUtils {
       sessionJarsPBLRsrc);
 
     String user = UserGroupInformation.getCurrentUser().getShortUserName();
-    ACLManager aclManager = new ACLManager(user, finalTezConf);
+    ACLManager aclManager = new ACLManager(user, amConfig.getTezConfiguration());
     Map<ApplicationAccessType, String> acls = aclManager.toYARNACls();
 
     if(dag != null) {
@@ -625,7 +613,7 @@ public class TezClientUtils {
     appContext.setAMContainerSpec(amContainer);
 
     appContext.setMaxAppAttempts(
-      finalTezConf.getInt(TezConfiguration.TEZ_AM_MAX_APP_ATTEMPTS,
+      amConfig.getTezConfiguration().getInt(TezConfiguration.TEZ_AM_MAX_APP_ATTEMPTS,
         TezConfiguration.TEZ_AM_MAX_APP_ATTEMPTS_DEFAULT));
 
     return appContext;
@@ -703,26 +691,16 @@ public class TezClientUtils {
         + "," + TezConstants.TEZ_CONTAINER_LOGGER_NAME);
   }
 
-  static Configuration createFinalTezConfForApp(TezConfiguration amConf) {
-    Configuration conf = new Configuration(false);
-    conf.setQuietMode(true);
-
+  static ConfigurationProto createFinalConfProtoForApp(Configuration amConf) {
     assert amConf != null;
-
-    Entry<String, String> entry;
-    Iterator<Entry<String, String>> iter = amConf.iterator();
-    while (iter.hasNext()) {
-      entry = iter.next();
-      // Copy all tez config parameters.
-      if (entry.getKey().startsWith(TezConfiguration.TEZ_PREFIX)) {
-        conf.set(entry.getKey(), entry.getValue());
-        if (LOG.isDebugEnabled()) {
-          LOG.debug("Adding tez dag am parameter from amConf: " + entry.getKey()
-              + ", with value: " + entry.getValue());
-        }
-      }
-    }
-    return conf;
+    ConfigurationProto.Builder builder = ConfigurationProto.newBuilder();
+    for (Entry<String, String> entry : amConf) {
+      PlanKeyValuePair.Builder kvp = PlanKeyValuePair.newBuilder();
+      kvp.setKey(entry.getKey());
+      kvp.setValue(entry.getValue());
+      builder.addConfKeyValues(kvp);
+    }
+    return builder.build();
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/tez/blob/fba3a512/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
index 8053283..21b9140 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
@@ -723,7 +723,6 @@ public class TezConfiguration extends Configuration {
 
   public static final int TEZ_AM_INLINE_TASK_EXECUTION_MAX_TASKS_DEFAULT = 1;
 
-
   // ACLs related configuration
   // Format supports a comma-separated list of users and groups with the users and groups
separated
   // by whitespace. e.g. "user1,user2 group1,group2"

http://git-wip-us.apache.org/repos/asf/tez/blob/fba3a512/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java
----------------------------------------------------------------------
diff --git a/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java b/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java
index 04c926e..ddd5963 100644
--- a/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java
+++ b/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java
@@ -17,13 +17,16 @@
  */
 package org.apache.tez.client;
 
+import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
 
 import java.io.File;
 import java.io.FileNotFoundException;
 import java.net.URL;
 import java.net.URLClassLoader;
+import java.util.HashMap;
 import java.util.Map;
 import java.util.Set;
 
@@ -39,6 +42,7 @@ import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.tez.dag.api.TezConfiguration;
 import org.apache.tez.dag.api.TezConstants;
 import org.apache.tez.dag.api.TezUncheckedException;
+import org.apache.tez.dag.api.records.DAGProtos;
 import org.junit.Assert;
 import org.junit.Test;
 
@@ -301,4 +305,27 @@ public class TestTezClientUtils {
     }
   }
 
+  @Test(timeout = 5000)
+  public void testConfigurationAllowAll() {
+    Configuration srcConf = new Configuration(false);
+
+    Map<String, String> confMap = new HashMap<String, String>();
+    confMap.put("ipc.timeout", "2000");
+    confMap.put("fs.defaultFS", "testfs:///");
+    confMap.put("tez.property", "tezProperty");
+    confMap.put("yarn.property", "yarnProperty");
+
+    for (Map.Entry<String, String> entry : confMap.entrySet()) {
+      srcConf.set(entry.getKey(), entry.getValue());
+    }
+
+    DAGProtos.ConfigurationProto confProto = TezClientUtils.createFinalConfProtoForApp(srcConf);
+
+    for (DAGProtos.PlanKeyValuePair kvPair : confProto.getConfKeyValuesList()) {
+      String val = confMap.remove(kvPair.getKey());
+      assertNotNull(val);
+      assertEquals(val, kvPair.getValue());
+    }
+    assertTrue(confMap.isEmpty());
+  }
 }


Mime
View raw message