tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ss...@apache.org
Subject git commit: TEZ-593. Add a config parameter to specify the YARN queue name for a DAG. (sseth)
Date Tue, 05 Nov 2013 21:52:48 GMT
Updated Branches:
  refs/heads/master 4eabbf714 -> 83a657bb5


TEZ-593. Add a config parameter to specify the YARN queue name for a
DAG. (sseth)


Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/83a657bb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/83a657bb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/83a657bb

Branch: refs/heads/master
Commit: 83a657bb5d88c1d080d45b19d5243990cfb67efa
Parents: 4eabbf7
Author: Siddharth Seth <sseth@apache.org>
Authored: Tue Nov 5 13:52:29 2013 -0800
Committer: Siddharth Seth <sseth@apache.org>
Committed: Tue Nov 5 13:52:29 2013 -0800

----------------------------------------------------------------------
 .../src/main/java/org/apache/tez/client/AMConfiguration.java   | 4 ++--
 .../src/main/java/org/apache/tez/client/TezClientUtils.java    | 4 +++-
 .../src/main/java/org/apache/tez/dag/api/TezConfiguration.java | 6 ++++++
 .../org/apache/tez/mapreduce/examples/FilterLinesByWord.java   | 2 +-
 .../java/org/apache/tez/mapreduce/examples/MRRSleepJob.java    | 2 +-
 .../org/apache/tez/mapreduce/examples/OrderedWordCount.java    | 2 +-
 .../main/java/org/apache/tez/mapreduce/client/YARNRunner.java  | 5 +++--
 .../java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java   | 2 ++
 .../test/java/org/apache/tez/mapreduce/TestMRRJobsDAGApi.java  | 4 ++--
 9 files changed, 21 insertions(+), 10 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/83a657bb/tez-api/src/main/java/org/apache/tez/client/AMConfiguration.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/client/AMConfiguration.java b/tez-api/src/main/java/org/apache/tez/client/AMConfiguration.java
index f452c74..3dd6424 100644
--- a/tez-api/src/main/java/org/apache/tez/client/AMConfiguration.java
+++ b/tez-api/src/main/java/org/apache/tez/client/AMConfiguration.java
@@ -38,15 +38,15 @@ public class AMConfiguration {
   private final TezConfiguration amConf;
   private final Credentials credentials;
 
-  public AMConfiguration(String queueName, Map<String, String> env,
+  public AMConfiguration(Map<String, String> env,
       Map<String, LocalResource> localResources,
       TezConfiguration conf, Credentials credentials) {
-    this.queueName = queueName;
     if (conf != null) {
       this.amConf = conf;
     } else {
       this.amConf = new TezConfiguration();
     }
+    this.queueName = this.amConf.get(TezConfiguration.TEZ_QUEUE_NAME);
 
     if (env != null) {
       this.env = env;

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/83a657bb/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
index 7209435..68394b9 100644
--- a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
+++ b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
@@ -428,7 +428,9 @@ public class TezClientUtils {
     appContext.setApplicationType(TezConfiguration.TEZ_APPLICATION_TYPE);
     appContext.setApplicationId(appId);
     appContext.setResource(capability);
-    appContext.setQueue(amConfig.getQueueName());
+    if (amConfig.getQueueName() != null) {
+      appContext.setQueue(amConfig.getQueueName());
+    }
     appContext.setApplicationName(amName);
     appContext.setCancelTokensWhenComplete(amConfig.getAMConf().getBoolean(
         TezConfiguration.TEZ_AM_CANCEL_DELEGATION_TOKEN,

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/83a657bb/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
index 59c93a3..6392ded 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
@@ -289,4 +289,10 @@ public class TezConfiguration extends Configuration {
   public static final String TEZ_PRE_WARM_PB_PLAN_BINARY_PATH =
       TEZ_SESSION_PRE_WARM_PREFIX + "dag-plan.pb.path";
 
+  /**
+   * The queue name for all jobs being submitted as part of a session, or for
+   * non session jobs.
+   */
+  public static final String TEZ_QUEUE_NAME = 
+      TEZ_PREFIX + "queue.name";
 }

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/83a657bb/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWord.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWord.java
b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWord.java
index 6067190..bd032e2 100644
--- a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWord.java
+++ b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWord.java
@@ -147,7 +147,7 @@ public class FilterLinesByWord {
 
 
 
-    AMConfiguration amConf = new AMConfiguration("default", null, commonLocalResources, tezConf,
null);
+    AMConfiguration amConf = new AMConfiguration(null, commonLocalResources, tezConf, null);
     TezSessionConfiguration sessionConf = new TezSessionConfiguration(amConf, tezConf);
     TezSession tezSession = new TezSession("FilterLinesByWordSession", sessionConf);
     tezSession.start(); // Why do I need to start the TezSession.

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/83a657bb/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/MRRSleepJob.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/MRRSleepJob.java
b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/MRRSleepJob.java
index 3c85122..8943dfa 100644
--- a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/MRRSleepJob.java
+++ b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/MRRSleepJob.java
@@ -827,7 +827,7 @@ public class MRRSleepJob extends Configured implements Tool {
     conf.set(TezConfiguration.TEZ_AM_JAVA_OPTS,
         MRHelpers.getMRAMJavaOpts(conf));
 
-    AMConfiguration amConfig = new AMConfiguration(null, null,
+    AMConfiguration amConfig = new AMConfiguration(null,
         null, conf, null);
 
     DAGClient dagClient =

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/83a657bb/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/OrderedWordCount.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/OrderedWordCount.java
b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/OrderedWordCount.java
index 1d20592..fa2533b 100644
--- a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/OrderedWordCount.java
+++ b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/OrderedWordCount.java
@@ -336,7 +336,7 @@ public class OrderedWordCount {
     // the tez jars.
 
     TezSession tezSession = null;
-    AMConfiguration amConfig = new AMConfiguration("default", null,
+    AMConfiguration amConfig = new AMConfiguration(null,
         null, tezConf, null);
     if (useTezSession) {
       LOG.info("Creating Tez Session");

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/83a657bb/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/YARNRunner.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/YARNRunner.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/YARNRunner.java
index 6b6dd35..d5b61b1 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/YARNRunner.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/YARNRunner.java
@@ -568,9 +568,10 @@ public class YARNRunner implements ClientProtocol {
     try {
       dagAMConf.set(TezConfiguration.TEZ_AM_STAGING_DIR,
           jobSubmitDir);
+      String queueName = jobConf.get(JobContext.QUEUE_NAME,
+          YarnConfiguration.DEFAULT_QUEUE_NAME);
+      dagAMConf.set(TezConfiguration.TEZ_QUEUE_NAME, queueName);
       AMConfiguration amConfig = new AMConfiguration(
-          jobConf.get(JobContext.QUEUE_NAME,
-              YarnConfiguration.DEFAULT_QUEUE_NAME),
           environment,
           jobLocalResources, dagAMConf, ts);
       tezClient.submitDAGApplication(appId, dag, amConfig);

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/83a657bb/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java
b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java
index f0cb289..2a25eac 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java
@@ -128,6 +128,8 @@ public class DeprecatedKeys {
   
   private static void populateMRToTezRuntimeParamMap() {
     
+    registerMRToRuntimeKeyTranslation(MRJobConfig.QUEUE_NAME, TezConfiguration.TEZ_QUEUE_NAME);
+    
     registerMRToRuntimeKeyTranslation(MRConfig.MAPRED_IFILE_READAHEAD, TezJobConfig.TEZ_RUNTIME_IFILE_READAHEAD);
 
     registerMRToRuntimeKeyTranslation(MRConfig.MAPRED_IFILE_READAHEAD_BYTES, TezJobConfig.TEZ_RUNTIME_IFILE_READAHEAD_BYTES);

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/83a657bb/tez-tests/src/test/java/org/apache/tez/mapreduce/TestMRRJobsDAGApi.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/test/java/org/apache/tez/mapreduce/TestMRRJobsDAGApi.java b/tez-tests/src/test/java/org/apache/tez/mapreduce/TestMRRJobsDAGApi.java
index 1bc7b4d..c2b6451 100644
--- a/tez-tests/src/test/java/org/apache/tez/mapreduce/TestMRRJobsDAGApi.java
+++ b/tez-tests/src/test/java/org/apache/tez/mapreduce/TestMRRJobsDAGApi.java
@@ -185,7 +185,7 @@ public class TestMRRJobsDAGApi {
         new HashMap<String, LocalResource>();
 
     AMConfiguration amConfig = new AMConfiguration(
-        "default", commonEnv, amLocalResources,
+        commonEnv, amLocalResources,
         tezConf, null);
     TezSessionConfiguration tezSessionConfig =
         new TezSessionConfiguration(amConfig, tezConf);
@@ -437,7 +437,7 @@ public class TestMRRJobsDAGApi {
     boolean reuseSession = reUseTezSession != null;
     TezSessionConfiguration tezSessionConfig;
     AMConfiguration amConfig = new AMConfiguration(
-        "default", commonEnv, amLocalResources,
+        commonEnv, amLocalResources,
         tezConf, null);
     if(!dagViaRPC) {
       // TODO Use utility method post TEZ-205 to figure out AM arguments etc.


Mime
View raw message