tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From bi...@apache.org
Subject git commit: TEZ-1041. Use VertexLocationHint consistently everywhere in the API (bikas)
Date Sun, 03 Aug 2014 00:39:05 GMT
Repository: tez
Updated Branches:
  refs/heads/master 57d342cc3 -> 0df3d0f6f


TEZ-1041. Use VertexLocationHint consistently everywhere in the API (bikas)


Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/0df3d0f6
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/0df3d0f6
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/0df3d0f6

Branch: refs/heads/master
Commit: 0df3d0f6f4f175dc013cc0286230eb5ddde98bed
Parents: 57d342c
Author: Bikas Saha <bikas@apache.org>
Authored: Sat Aug 2 17:39:12 2014 -0700
Committer: Bikas Saha <bikas@apache.org>
Committed: Sat Aug 2 17:39:12 2014 -0700

----------------------------------------------------------------------
 CHANGES.txt                                              |  1 +
 tez-api/src/main/java/org/apache/tez/dag/api/DAG.java    |  6 +++---
 tez-api/src/main/java/org/apache/tez/dag/api/Vertex.java | 11 ++++++-----
 .../main/java/org/apache/tez/dag/app/DAGAppMaster.java   |  5 +----
 .../apache/tez/mapreduce/examples/FilterLinesByWord.java |  3 ++-
 .../mapreduce/examples/FilterLinesByWordOneToOne.java    |  3 ++-
 .../org/apache/tez/mapreduce/examples/MRRSleepJob.java   |  3 ++-
 .../apache/tez/mapreduce/examples/OrderedWordCount.java  |  2 +-
 .../java/org/apache/tez/mapreduce/client/YARNRunner.java |  3 ++-
 .../java/org/apache/tez/mapreduce/TestMRRJobsDAGApi.java |  3 ++-
 10 files changed, 22 insertions(+), 18 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tez/blob/0df3d0f6/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index e2e673f..c862935 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -40,6 +40,7 @@ INCOMPATIBLE CHANGES
   to require constructors for creation, and remove the initialize methods.
   TEZ-1133. Remove some unused methods from MRHelpers.
   TEZ-1346. Change Processor to require context constructors for creation, and remove the
requirement of the initialize method requiring the context.
+  TEZ-1041. Use VertexLocationHint consistently everywhere in the API
 
 Release 0.4.0-incubating: 2014-04-05
 

http://git-wip-us.apache.org/repos/asf/tez/blob/0df3d0f6/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java b/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java
index 7e35c57..15d23e3 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java
@@ -587,9 +587,9 @@ public class DAG {
         taskConfigBuilder.addEnvironmentSetting(envSettingBuilder);
       }
 
-      if (vertex.getTaskLocationsHint() != null) {
-        if (vertex.getTaskLocationsHint().getTaskLocationHints() != null) {
-          for (TaskLocationHint hint : vertex.getTaskLocationsHint().getTaskLocationHints())
{
+      if (vertex.getLocationHint() != null) {
+        if (vertex.getLocationHint().getTaskLocationHints() != null) {
+          for (TaskLocationHint hint : vertex.getLocationHint().getTaskLocationHints()) {
             PlanTaskLocationHint.Builder taskLocationHintBuilder = PlanTaskLocationHint.newBuilder();
 
             if (hint.getAffinitizedContainer() != null) {

http://git-wip-us.apache.org/repos/asf/tez/blob/0df3d0f6/tez-api/src/main/java/org/apache/tez/dag/api/Vertex.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/Vertex.java b/tez-api/src/main/java/org/apache/tez/dag/api/Vertex.java
index 664df46..25a3990 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/Vertex.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/Vertex.java
@@ -43,7 +43,7 @@ public class Vertex {
   private final ProcessorDescriptor processorDescriptor;
 
   private int parallelism;
-  private VertexLocationHint taskLocationsHint;
+  private VertexLocationHint locationHint;
   private final Resource taskResource;
   private Map<String, LocalResource> taskLocalResources = new HashMap<String, LocalResource>();
   private Map<String, String> taskEnvironment = new HashMap<String, String>();
@@ -139,19 +139,20 @@ public class Vertex {
    * @param locations list of locations for each task in the vertex
    * @return this Vertex
    */
-  public Vertex setTaskLocationsHint(List<TaskLocationHint> locations) {
+  public Vertex setLocationHint(VertexLocationHint locationHint) {
+    List<TaskLocationHint> locations = locationHint.getTaskLocationHints();
     if (locations == null) {
       return this;
     }
     Preconditions.checkArgument((locations.size() == parallelism), 
         "Locations array length must match the parallelism set for the vertex");
-    taskLocationsHint = new VertexLocationHint(locations);
+    this.locationHint = locationHint;
     return this;
   }
 
   // used internally to create parallelism location resource file
-  VertexLocationHint getTaskLocationsHint() {
-    return taskLocationsHint;
+  VertexLocationHint getLocationHint() {
+    return locationHint;
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/tez/blob/0df3d0f6/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
index 132e5fa..0626fc8 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
@@ -46,7 +46,6 @@ import java.util.Map.Entry;
 import java.util.Set;
 import java.util.Timer;
 import java.util.TimerTask;
-import java.util.UUID;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.locks.Lock;
@@ -62,7 +61,6 @@ import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.Text;
 import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
 import org.apache.hadoop.security.Credentials;
 import org.apache.hadoop.security.UserGroupInformation;
@@ -1020,8 +1018,7 @@ public class DAGAppMaster extends AbstractService {
       preWarmVertex.setTaskLocalFiles(preWarmContext.getLocalResources());
     }
     if (preWarmContext.getLocationHints() != null) {
-      preWarmVertex.setTaskLocationsHint(
-        preWarmContext.getLocationHints().getTaskLocationHints());
+      preWarmVertex.setLocationHint(preWarmContext.getLocationHints());
     }
     if (preWarmContext.getJavaOpts() != null) {
       preWarmVertex.setTaskLaunchCmdOpts(preWarmContext.getJavaOpts());

http://git-wip-us.apache.org/repos/asf/tez/blob/0df3d0f6/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWord.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWord.java
b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWord.java
index 9588c72..465fec2 100644
--- a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWord.java
+++ b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWord.java
@@ -64,6 +64,7 @@ import org.apache.tez.dag.api.TezConfiguration;
 import org.apache.tez.dag.api.TezException;
 import org.apache.tez.dag.api.TezUncheckedException;
 import org.apache.tez.dag.api.Vertex;
+import org.apache.tez.dag.api.VertexLocationHint;
 import org.apache.tez.dag.api.client.DAGClient;
 import org.apache.tez.dag.api.client.DAGStatus;
 import org.apache.tez.dag.api.client.StatusGetOpts;
@@ -187,7 +188,7 @@ public class FilterLinesByWord extends Configured implements Tool {
         FilterByWordInputProcessor.class.getName()).setUserPayload(stage1Payload),
         stage1NumTasks, MRHelpers.getMapResource(stage1Conf));
     if (generateSplitsInClient) {
-      stage1Vertex.setTaskLocationsHint(inputSplitInfo.getTaskLocationHints());
+      stage1Vertex.setLocationHint(new VertexLocationHint(inputSplitInfo.getTaskLocationHints()));
       Map<String, LocalResource> stage1LocalResources = new HashMap<String, LocalResource>();
       stage1LocalResources.putAll(commonLocalResources);
       MRHelpers.updateLocalResourcesForInputSplits(fs, inputSplitInfo, stage1LocalResources);

http://git-wip-us.apache.org/repos/asf/tez/blob/0df3d0f6/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWordOneToOne.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWordOneToOne.java
b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWordOneToOne.java
index 9143351..83be555 100644
--- a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWordOneToOne.java
+++ b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWordOneToOne.java
@@ -56,6 +56,7 @@ import org.apache.tez.dag.api.TezConfiguration;
 import org.apache.tez.dag.api.TezException;
 import org.apache.tez.dag.api.TezUncheckedException;
 import org.apache.tez.dag.api.Vertex;
+import org.apache.tez.dag.api.VertexLocationHint;
 import org.apache.tez.dag.api.client.DAGClient;
 import org.apache.tez.dag.api.client.DAGStatus;
 import org.apache.tez.mapreduce.committer.MROutputCommitter;
@@ -173,7 +174,7 @@ public class FilterLinesByWordOneToOne extends Configured implements Tool
{
         FilterByWordInputProcessor.class.getName()).setUserPayload(stage1Payload),
         stage1NumTasks, MRHelpers.getMapResource(stage1Conf));
     if (generateSplitsInClient) {
-      stage1Vertex.setTaskLocationsHint(inputSplitInfo.getTaskLocationHints());
+      stage1Vertex.setLocationHint(new VertexLocationHint(inputSplitInfo.getTaskLocationHints()));
       Map<String, LocalResource> stage1LocalResources = new HashMap<String, LocalResource>();
       stage1LocalResources.putAll(commonLocalResources);
       MRHelpers.updateLocalResourcesForInputSplits(fs, inputSplitInfo, stage1LocalResources);

http://git-wip-us.apache.org/repos/asf/tez/blob/0df3d0f6/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/MRRSleepJob.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/MRRSleepJob.java
b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/MRRSleepJob.java
index ec49a1c..ab75441 100644
--- a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/MRRSleepJob.java
+++ b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/MRRSleepJob.java
@@ -69,6 +69,7 @@ import org.apache.tez.dag.api.ProcessorDescriptor;
 import org.apache.tez.dag.api.TezConfiguration;
 import org.apache.tez.dag.api.TezUncheckedException;
 import org.apache.tez.dag.api.Vertex;
+import org.apache.tez.dag.api.VertexLocationHint;
 import org.apache.tez.dag.api.client.DAGClient;
 import org.apache.tez.dag.api.client.DAGStatus;
 import org.apache.tez.mapreduce.common.MRInputAMSplitGenerator;
@@ -546,7 +547,7 @@ public class MRRSleepJob extends Configured implements Tool {
         MapProcessor.class.getName()).setUserPayload(mapUserPayload),
         numTasks, MRHelpers.getMapResource(mapStageConf));
     if (!generateSplitsInAM) {
-      mapVertex.setTaskLocationsHint(inputSplitInfo.getTaskLocationHints());
+      mapVertex.setLocationHint(new VertexLocationHint(inputSplitInfo.getTaskLocationHints()));
     }
     
     if (writeSplitsToDFS) {

http://git-wip-us.apache.org/repos/asf/tez/blob/0df3d0f6/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/OrderedWordCount.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/OrderedWordCount.java
b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/OrderedWordCount.java
index 7af5402..14240ea 100644
--- a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/OrderedWordCount.java
+++ b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/OrderedWordCount.java
@@ -216,7 +216,7 @@ public class OrderedWordCount extends Configured implements Tool {
             .setHistoryText(mapStageHistoryText),
         numMaps, MRHelpers.getMapResource(mapStageConf));
     if (generateSplitsInClient) {
-      mapVertex.setTaskLocationsHint(inputSplitInfo.getTaskLocationHints());
+      mapVertex.setLocationHint(new VertexLocationHint(inputSplitInfo.getTaskLocationHints()));
       Map<String, LocalResource> mapLocalResources =
           new HashMap<String, LocalResource>();
       mapLocalResources.putAll(commonLocalResources);

http://git-wip-us.apache.org/repos/asf/tez/blob/0df3d0f6/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/YARNRunner.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/YARNRunner.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/YARNRunner.java
index d785be3..2bc2ff0 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/YARNRunner.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/YARNRunner.java
@@ -84,6 +84,7 @@ import org.apache.tez.dag.api.ProcessorDescriptor;
 import org.apache.tez.dag.api.TezConfiguration;
 import org.apache.tez.dag.api.TezException;
 import org.apache.tez.dag.api.Vertex;
+import org.apache.tez.dag.api.VertexLocationHint;
 import org.apache.tez.dag.api.VertexLocationHint.TaskLocationHint;
 import org.apache.tez.dag.api.VertexManagerPluginDescriptor;
 import org.apache.tez.dag.api.client.DAGStatus;
@@ -424,7 +425,7 @@ public class YARNRunner implements ClientProtocol {
 
     vertex.setTaskEnvironment(taskEnv)
         .setTaskLocalFiles(taskLocalResources)
-        .setTaskLocationsHint(locations)
+        .setLocationHint(new VertexLocationHint(locations))
         .setTaskLaunchCmdOpts(taskJavaOpts);
     
     if (!isMap) {

http://git-wip-us.apache.org/repos/asf/tez/blob/0df3d0f6/tez-tests/src/test/java/org/apache/tez/mapreduce/TestMRRJobsDAGApi.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/test/java/org/apache/tez/mapreduce/TestMRRJobsDAGApi.java b/tez-tests/src/test/java/org/apache/tez/mapreduce/TestMRRJobsDAGApi.java
index b0d5061..44f1343 100644
--- a/tez-tests/src/test/java/org/apache/tez/mapreduce/TestMRRJobsDAGApi.java
+++ b/tez-tests/src/test/java/org/apache/tez/mapreduce/TestMRRJobsDAGApi.java
@@ -82,6 +82,7 @@ import org.apache.tez.dag.api.TezConfiguration;
 import org.apache.tez.dag.api.TezException;
 import org.apache.tez.dag.api.TezUncheckedException;
 import org.apache.tez.dag.api.Vertex;
+import org.apache.tez.dag.api.VertexLocationHint;
 import org.apache.tez.dag.api.client.DAGClient;
 import org.apache.tez.dag.api.client.DAGStatus;
 import org.apache.tez.dag.api.client.DAGStatus.State;
@@ -497,7 +498,7 @@ public class TestMRRJobsDAGApi {
               LocalResourceType.FILE, LocalResourceVisibility.APPLICATION));
 
       stage1Vertex.setTaskLocalFiles(stage1LocalResources);
-      stage1Vertex.setTaskLocationsHint(inputSplitInfo.getTaskLocationHints());
+      stage1Vertex.setLocationHint(new VertexLocationHint(inputSplitInfo.getTaskLocationHints()));
     }
     
     // TODO env, resources


Mime
View raw message