tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hit...@apache.org
Subject tez git commit: TEZ-2385. branch-0.6 compile failure caused by TEZ-2226. (hitesh)
Date Wed, 29 Apr 2015 21:38:57 GMT
Repository: tez
Updated Branches:
  refs/heads/branch-0.6 da85ca8cf -> a03e1c951


TEZ-2385. branch-0.6 compile failure caused by TEZ-2226. (hitesh)


Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/a03e1c95
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/a03e1c95
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/a03e1c95

Branch: refs/heads/branch-0.6
Commit: a03e1c951782372857a2431c41bef570b3ac8e10
Parents: da85ca8
Author: Hitesh Shah <hitesh@apache.org>
Authored: Wed Apr 29 14:38:40 2015 -0700
Committer: Hitesh Shah <hitesh@apache.org>
Committed: Wed Apr 29 14:38:40 2015 -0700

----------------------------------------------------------------------
 CHANGES.txt                                     |  1 +
 .../main/java/org/apache/tez/dag/api/DAG.java   | 34 ++++++++++++++------
 .../apache/tez/dag/api/TezConfiguration.java    |  2 --
 3 files changed, 25 insertions(+), 12 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tez/blob/a03e1c95/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 5ba43b6..8b8d4f5 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -6,6 +6,7 @@ Release 0.6.1: Unreleased
 INCOMPATIBLE CHANGES
 
 ALL CHANGES:
+  TEZ-2385. branch-0.6 compile failure caused by TEZ-2226.
   TEZ-2226. Disable writing history to timeline if domain creation fails.
   TEZ-2259. Push additional data to Timeline for Recovery for better consumption in UI.
   TEZ-2365. Update tez-ui war's license/notice to reflect OFL license correctly.

http://git-wip-us.apache.org/repos/asf/tez/blob/a03e1c95/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java b/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java
index 2779286..906b1b7 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java
@@ -84,6 +84,7 @@ public class DAG {
   Credentials credentials = new Credentials();
   Set<VertexGroup> vertexGroups = Sets.newHashSet();
   Set<GroupInputEdge> groupInputEdges = Sets.newHashSet();
+  private Map<String, String> dagConf = new HashMap<String, String>();
 
   private DAGAccessControls dagAccessControls;
   Map<String, LocalResource> commonTaskLocalFiles = Maps.newHashMap();
@@ -319,7 +320,12 @@ public class DAG {
   public String getName() {
     return this.name;
   }
-  
+
+  public DAG setConf(String property, String value) {
+    dagConf.put(property, value);
+    return this;
+  }
+
   @Private
   @VisibleForTesting
   public Map<String,String> getDagConf() {
@@ -670,16 +676,16 @@ public class DAG {
   }
 
   // create protobuf message describing DAG
-  public DAGPlan createDag(Configuration dagConf, Credentials extraCredentials,
+  public DAGPlan createDag(Configuration conf, Credentials extraCredentials,
                            Map<String, LocalResource> tezJarResources, LocalResource
binaryConfig,
                            boolean tezLrsAsArchive) {
-    return createDag(dagConf, extraCredentials, tezJarResources, binaryConfig, tezLrsAsArchive,
+    return createDag(conf, extraCredentials, tezJarResources, binaryConfig, tezLrsAsArchive,
         null);
   }
 
   // create protobuf message describing DAG
   @Private
-  public DAGPlan createDag(Configuration dagConf, Credentials extraCredentials,
+  public DAGPlan createDag(Configuration tezConf, Credentials extraCredentials,
       Map<String, LocalResource> tezJarResources, LocalResource binaryConfig,
       boolean tezLrsAsArchive, Map<String, String> additionalConfigs) {
     verify(true);
@@ -728,9 +734,9 @@ public class DAG {
       // infer credentials, resources and parallelism from data source
       Resource vertexTaskResource = vertex.getTaskResource();
       if (vertexTaskResource == null) {
-        vertexTaskResource = Resource.newInstance(dagConf.getInt(
+        vertexTaskResource = Resource.newInstance(tezConf.getInt(
             TezConfiguration.TEZ_TASK_RESOURCE_MEMORY_MB,
-            TezConfiguration.TEZ_TASK_RESOURCE_MEMORY_MB_DEFAULT), dagConf.getInt(
+            TezConfiguration.TEZ_TASK_RESOURCE_MEMORY_MB_DEFAULT), tezConf.getInt(
             TezConfiguration.TEZ_TASK_RESOURCE_CPU_VCORES,
             TezConfiguration.TEZ_TASK_RESOURCE_CPU_VCORES_DEFAULT));
       }
@@ -801,7 +807,7 @@ public class DAG {
       taskConfigBuilder.setMemoryMb(vertexTaskResource.getMemory());
       taskConfigBuilder.setVirtualCores(vertexTaskResource.getVirtualCores());
       taskConfigBuilder.setJavaOpts(
-          TezClientUtils.addDefaultsToTaskLaunchCmdOpts(vertex.getTaskLaunchCmdOpts(), dagConf));
+          TezClientUtils.addDefaultsToTaskLaunchCmdOpts(vertex.getTaskLaunchCmdOpts(), tezConf));
 
       taskConfigBuilder.setTaskModule(vertex.getName());
       if (!vertexLRs.isEmpty()) {
@@ -809,7 +815,7 @@ public class DAG {
       }
 
       Map<String, String> taskEnv = Maps.newHashMap(vertex.getTaskEnvironment());
-      TezYARNUtils.setupDefaultEnv(taskEnv, dagConf,
+      TezYARNUtils.setupDefaultEnv(taskEnv, tezConf,
           TezConfiguration.TEZ_TASK_LAUNCH_ENV,
           TezConfiguration.TEZ_TASK_LAUNCH_ENV_DEFAULT, tezLrsAsArchive);
       for (Map.Entry<String, String> entry : taskEnv.entrySet()) {
@@ -879,8 +885,8 @@ public class DAG {
 
     ConfigurationProto.Builder confProtoBuilder =
         ConfigurationProto.newBuilder();
-    if (dagConf != null) {
-      Iterator<Entry<String, String>> iter = dagConf.iterator();
+    if (tezConf != null) {
+      Iterator<Entry<String, String>> iter = tezConf.iterator();
       while (iter.hasNext()) {
         Entry<String, String> entry = iter.next();
         PlanKeyValuePair.Builder kvp = PlanKeyValuePair.newBuilder();
@@ -909,6 +915,14 @@ public class DAG {
         confProtoBuilder.addConfKeyValues(kvp);
       }
     }
+    if (this.dagConf != null && !this.dagConf.isEmpty()) {
+      for (Entry<String, String> entry : this.dagConf.entrySet()) {
+        PlanKeyValuePair.Builder kvp = PlanKeyValuePair.newBuilder();
+        kvp.setKey(entry.getKey());
+        kvp.setValue(entry.getValue());
+        confProtoBuilder.addConfKeyValues(kvp);
+      }
+    }
     dagBuilder.setDagKeyValues(confProtoBuilder); // This does not seem to be used anywhere
     // should this replace BINARY_PB_CONF???
 

http://git-wip-us.apache.org/repos/asf/tez/blob/a03e1c95/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
index c5a3615..debc269 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
@@ -170,7 +170,6 @@ public class TezConfiguration extends Configuration {
   /**
    * Boolean value. Determine whether to log history events per dag
    */
-  @ConfigurationScope(Scope.DAG)
   @Private
   public static final String TEZ_DAG_HISTORY_LOGGING_ENABLED =
       TEZ_PREFIX + "dag.history.logging.enabled";
@@ -631,7 +630,6 @@ public class TezConfiguration extends Configuration {
    * Boolean value. Allow/disable logging for all dags in a session   
    */
   @Private
-  @ConfigurationScope(Scope.AM)
   public static final String TEZ_AM_HISTORY_LOGGING_ENABLED =
       TEZ_AM_PREFIX + "history.logging.enabled";
   public static final boolean TEZ_AM_HISTORY_LOGGING_ENABLED_DEFAULT = true;


Mime
View raw message