tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hit...@apache.org
Subject tez git commit: TEZ-2627. Support for Tez Job Priorities. (Saikat via hitesh)
Date Mon, 27 Jul 2015 22:48:44 GMT
Repository: tez
Updated Branches:
  refs/heads/master b04676932 -> cf400882b


TEZ-2627. Support for Tez Job Priorities. (Saikat via hitesh)


Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/cf400882
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/cf400882
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/cf400882

Branch: refs/heads/master
Commit: cf400882bdc2247f3cf923d571c0863c6a556dd1
Parents: b046769
Author: Hitesh Shah <hitesh@apache.org>
Authored: Mon Jul 27 15:48:14 2015 -0700
Committer: Hitesh Shah <hitesh@apache.org>
Committed: Mon Jul 27 15:48:14 2015 -0700

----------------------------------------------------------------------
 CHANGES.txt                                     |  1 +
 .../org/apache/tez/client/TezClientUtils.java   | 21 +++++++++++
 .../apache/tez/dag/api/TezConfiguration.java    |  6 ++++
 .../apache/tez/client/TestTezClientUtils.java   | 38 +++++++++++++++++++-
 .../java/org/apache/tez/test/TestTezJobs.java   |  1 +
 5 files changed, 66 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tez/blob/cf400882/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 6d751ce..f7f2e76 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -11,6 +11,7 @@ INCOMPATIBLE CHANGES
   TEZ-2468. Change the minimum Java version to Java 7.
 
 ALL CHANGES:
+  TEZ-2627. Support for Tez Job Priorities.
   TEZ-2623. Fix module dependencies related to hadoop-auth.
   TEZ-2464. Move older releases to dist archive.
   TEZ-2239. Update Tez UI docs to explain how to configure history url for YARN.

http://git-wip-us.apache.org/repos/asf/tez/blob/cf400882/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
index 26d5e2a..8bfaa1f 100644
--- a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
+++ b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
@@ -68,6 +68,7 @@ import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
 import org.apache.hadoop.yarn.api.records.LocalResource;
 import org.apache.hadoop.yarn.api.records.LocalResourceType;
 import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
+import org.apache.hadoop.yarn.api.records.Priority;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.api.records.YarnApplicationState;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -662,6 +663,8 @@ public class TezClientUtils {
     if (amConfig.getQueueName() != null) {
       appContext.setQueue(amConfig.getQueueName());
     }
+    // set the application priority
+    setApplicationPriority(appContext, amConfig);
     appContext.setApplicationName(amName);
     appContext.setCancelTokensWhenComplete(amConfig.getTezConfiguration().getBoolean(
         TezConfiguration.TEZ_CANCEL_DELEGATION_TOKENS_ON_COMPLETION,
@@ -1008,4 +1011,22 @@ public class TezClientUtils {
       return null;
     }
   }
+
+  @VisibleForTesting
+  // this method will set the app priority only if the priority config has been defined
+  public static void setApplicationPriority(ApplicationSubmissionContext context,
+      AMConfiguration amConfig) {
+    if (amConfig.getTezConfiguration().get(TezConfiguration.TEZ_AM_APPLICATION_PRIORITY)
!= null) {
+      // since we already checked not null condition, we are guaranteed that default value
+      // (0 in this case for getInt) will not be returned/used.
+      // The idea is to not use any default priority from TEZ side, if none provided in config;
+      // let YARN determine the priority of the submitted application
+      int priority = amConfig.getTezConfiguration().getInt(TezConfiguration.TEZ_AM_APPLICATION_PRIORITY,
0);
+      context.setPriority(Priority.newInstance(priority));
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("Settting TEZ application priority, applicationId= " + context.getApplicationId()
+
+            ", priority= " + context.getPriority().getPriority());
+      }
+    }
+  }
 }

http://git-wip-us.apache.org/repos/asf/tez/blob/cf400882/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
index 2114793..3b7378a 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
@@ -912,6 +912,12 @@ public class TezConfiguration extends Configuration {
   @ConfigurationScope(Scope.AM)
   public static final String TEZ_QUEUE_NAME = TEZ_PREFIX + "queue.name";
 
+  /**
+   * String value. The job priority of submitted application.
+   */
+  @ConfigurationScope(Scope.AM)
+  public static final String TEZ_AM_APPLICATION_PRIORITY = TEZ_PREFIX + "am.application.priority";
+
   @Unstable
   /**
    * Boolean value. Generate debug artifacts like DAG plan in text format.

http://git-wip-us.apache.org/repos/asf/tez/blob/cf400882/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java
----------------------------------------------------------------------
diff --git a/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java b/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java
index cfba917..2d4e005 100644
--- a/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java
+++ b/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java
@@ -57,6 +57,7 @@ import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.util.Records;
 import org.apache.tez.common.security.HistoryACLPolicyManager;
 import org.apache.tez.common.security.JobTokenIdentifier;
 import org.apache.tez.common.security.JobTokenSecretManager;
@@ -71,7 +72,6 @@ import org.apache.tez.dag.api.records.DAGProtos.ConfigurationProto;
 import org.apache.tez.dag.api.records.DAGProtos.PlanKeyValuePair;
 import org.junit.Assert;
 import org.junit.Test;
-
 /**
  * 
  */
@@ -204,6 +204,42 @@ public class TestTezClientUtils {
     assertFalse(localizedMap.isEmpty());
   }
 
+  @Test(timeout = 2000)
+  // this test checks if the priority field is set properly in the
+  // ApplicationSubmissionContext
+  public void testAppSubmissionContextForPriority() throws Exception {
+    TezConfiguration tezConf = new TezConfiguration();
+    int testpriority = 999;
+    ApplicationId appId = ApplicationId.newInstance(1000, 1);
+    Credentials credentials = new Credentials();
+    TezClientUtils.createSessionToken(appId.toString(),
+        new JobTokenSecretManager(), credentials);
+    tezConf.setBoolean(TezConfiguration.TEZ_IGNORE_LIB_URIS, true);
+    Map<String, LocalResource> m = new HashMap<String, LocalResource>();
+    tezConf.setInt(TezConfiguration.TEZ_AM_APPLICATION_PRIORITY, testpriority);
+    AMConfiguration amConf =
+        new AMConfiguration(tezConf, new HashMap<String, LocalResource>(), credentials);
+    ApplicationSubmissionContext appcontext;
+    appcontext = TezClientUtils.createApplicationSubmissionContext(
+        appId, null, "dagname",
+        amConf, m,
+        credentials, false,
+        new TezApiVersionInfo(), null);
+    assertEquals(testpriority, appcontext.getPriority().getPriority());
+  }
+
+  @Test(timeout=1000)
+  // when tez config property for app priority not set,
+  // tez should not set app priority and let YARN deal with it.
+  public void testSetApplicationPriority() {
+    TezConfiguration conf = new TezConfiguration(false);
+    AMConfiguration amconfig = new AMConfiguration(conf, null, null);
+    ApplicationSubmissionContext appContext = Records
+        .newRecord(ApplicationSubmissionContext.class);
+    TezClientUtils.setApplicationPriority(appContext, amconfig);
+    assertNull(appContext.getPriority());
+  }
+
   @Test(timeout = 5000)
   public void testSessionTokenInAmClc() throws IOException, YarnException {
 

http://git-wip-us.apache.org/repos/asf/tez/blob/cf400882/tez-tests/src/test/java/org/apache/tez/test/TestTezJobs.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/test/java/org/apache/tez/test/TestTezJobs.java b/tez-tests/src/test/java/org/apache/tez/test/TestTezJobs.java
index 0d0eaac..ab89ddb 100644
--- a/tez-tests/src/test/java/org/apache/tez/test/TestTezJobs.java
+++ b/tez-tests/src/test/java/org/apache/tez/test/TestTezJobs.java
@@ -299,6 +299,7 @@ public class TestTezJobs {
 
     String[] args = new String[] {
         "-D" + TezConfiguration.TEZ_AM_STAGING_DIR + "=" + stagingDirPath.toString(),
+        "-D" + TezConfiguration.TEZ_AM_APPLICATION_PRIORITY + "=" + "2",
         "-counter", inPath1.toString(), inPath2.toString(), "1", outPath.toString() };
     assertEquals(0, sortMergeJoinExample.run(args));
 


Mime
View raw message