Return-Path: X-Original-To: apmail-tez-commits-archive@minotaur.apache.org Delivered-To: apmail-tez-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 021F411B70 for ; Sun, 3 Aug 2014 00:39:06 +0000 (UTC) Received: (qmail 20770 invoked by uid 500); 3 Aug 2014 00:39:05 -0000 Delivered-To: apmail-tez-commits-archive@tez.apache.org Received: (qmail 20732 invoked by uid 500); 3 Aug 2014 00:39:05 -0000 Mailing-List: contact commits-help@tez.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@tez.apache.org Delivered-To: mailing list commits@tez.apache.org Received: (qmail 20723 invoked by uid 99); 3 Aug 2014 00:39:05 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Sun, 03 Aug 2014 00:39:05 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id 4DD0281BFF3; Sun, 3 Aug 2014 00:39:05 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: bikas@apache.org To: commits@tez.apache.org Message-Id: <0623c403ca6346b5aeda2f73052d5eb0@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: git commit: TEZ-1041. Use VertexLocationHint consistently everywhere in the API (bikas) Date: Sun, 3 Aug 2014 00:39:05 +0000 (UTC) Repository: tez Updated Branches: refs/heads/master 57d342cc3 -> 0df3d0f6f TEZ-1041. Use VertexLocationHint consistently everywhere in the API (bikas) Project: http://git-wip-us.apache.org/repos/asf/tez/repo Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/0df3d0f6 Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/0df3d0f6 Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/0df3d0f6 Branch: refs/heads/master Commit: 0df3d0f6f4f175dc013cc0286230eb5ddde98bed Parents: 57d342c Author: Bikas Saha Authored: Sat Aug 2 17:39:12 2014 -0700 Committer: Bikas Saha Committed: Sat Aug 2 17:39:12 2014 -0700 ---------------------------------------------------------------------- CHANGES.txt | 1 + tez-api/src/main/java/org/apache/tez/dag/api/DAG.java | 6 +++--- tez-api/src/main/java/org/apache/tez/dag/api/Vertex.java | 11 ++++++----- .../main/java/org/apache/tez/dag/app/DAGAppMaster.java | 5 +---- .../apache/tez/mapreduce/examples/FilterLinesByWord.java | 3 ++- .../mapreduce/examples/FilterLinesByWordOneToOne.java | 3 ++- .../org/apache/tez/mapreduce/examples/MRRSleepJob.java | 3 ++- .../apache/tez/mapreduce/examples/OrderedWordCount.java | 2 +- .../java/org/apache/tez/mapreduce/client/YARNRunner.java | 3 ++- .../java/org/apache/tez/mapreduce/TestMRRJobsDAGApi.java | 3 ++- 10 files changed, 22 insertions(+), 18 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tez/blob/0df3d0f6/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index e2e673f..c862935 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -40,6 +40,7 @@ INCOMPATIBLE CHANGES to require constructors for creation, and remove the initialize methods. TEZ-1133. Remove some unused methods from MRHelpers. TEZ-1346. Change Processor to require context constructors for creation, and remove the requirement of the initialize method requiring the context. + TEZ-1041. Use VertexLocationHint consistently everywhere in the API Release 0.4.0-incubating: 2014-04-05 http://git-wip-us.apache.org/repos/asf/tez/blob/0df3d0f6/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java ---------------------------------------------------------------------- diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java b/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java index 7e35c57..15d23e3 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java @@ -587,9 +587,9 @@ public class DAG { taskConfigBuilder.addEnvironmentSetting(envSettingBuilder); } - if (vertex.getTaskLocationsHint() != null) { - if (vertex.getTaskLocationsHint().getTaskLocationHints() != null) { - for (TaskLocationHint hint : vertex.getTaskLocationsHint().getTaskLocationHints()) { + if (vertex.getLocationHint() != null) { + if (vertex.getLocationHint().getTaskLocationHints() != null) { + for (TaskLocationHint hint : vertex.getLocationHint().getTaskLocationHints()) { PlanTaskLocationHint.Builder taskLocationHintBuilder = PlanTaskLocationHint.newBuilder(); if (hint.getAffinitizedContainer() != null) { http://git-wip-us.apache.org/repos/asf/tez/blob/0df3d0f6/tez-api/src/main/java/org/apache/tez/dag/api/Vertex.java ---------------------------------------------------------------------- diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/Vertex.java b/tez-api/src/main/java/org/apache/tez/dag/api/Vertex.java index 664df46..25a3990 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/Vertex.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/Vertex.java @@ -43,7 +43,7 @@ public class Vertex { private final ProcessorDescriptor processorDescriptor; private int parallelism; - private VertexLocationHint taskLocationsHint; + private VertexLocationHint locationHint; private final Resource taskResource; private Map taskLocalResources = new HashMap(); private Map taskEnvironment = new HashMap(); @@ -139,19 +139,20 @@ public class Vertex { * @param locations list of locations for each task in the vertex * @return this Vertex */ - public Vertex setTaskLocationsHint(List locations) { + public Vertex setLocationHint(VertexLocationHint locationHint) { + List locations = locationHint.getTaskLocationHints(); if (locations == null) { return this; } Preconditions.checkArgument((locations.size() == parallelism), "Locations array length must match the parallelism set for the vertex"); - taskLocationsHint = new VertexLocationHint(locations); + this.locationHint = locationHint; return this; } // used internally to create parallelism location resource file - VertexLocationHint getTaskLocationsHint() { - return taskLocationsHint; + VertexLocationHint getLocationHint() { + return locationHint; } /** http://git-wip-us.apache.org/repos/asf/tez/blob/0df3d0f6/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java ---------------------------------------------------------------------- diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java index 132e5fa..0626fc8 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java @@ -46,7 +46,6 @@ import java.util.Map.Entry; import java.util.Set; import java.util.Timer; import java.util.TimerTask; -import java.util.UUID; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.locks.Lock; @@ -62,7 +61,6 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.io.Text; import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.UserGroupInformation; @@ -1020,8 +1018,7 @@ public class DAGAppMaster extends AbstractService { preWarmVertex.setTaskLocalFiles(preWarmContext.getLocalResources()); } if (preWarmContext.getLocationHints() != null) { - preWarmVertex.setTaskLocationsHint( - preWarmContext.getLocationHints().getTaskLocationHints()); + preWarmVertex.setLocationHint(preWarmContext.getLocationHints()); } if (preWarmContext.getJavaOpts() != null) { preWarmVertex.setTaskLaunchCmdOpts(preWarmContext.getJavaOpts()); http://git-wip-us.apache.org/repos/asf/tez/blob/0df3d0f6/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWord.java ---------------------------------------------------------------------- diff --git a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWord.java b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWord.java index 9588c72..465fec2 100644 --- a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWord.java +++ b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWord.java @@ -64,6 +64,7 @@ import org.apache.tez.dag.api.TezConfiguration; import org.apache.tez.dag.api.TezException; import org.apache.tez.dag.api.TezUncheckedException; import org.apache.tez.dag.api.Vertex; +import org.apache.tez.dag.api.VertexLocationHint; import org.apache.tez.dag.api.client.DAGClient; import org.apache.tez.dag.api.client.DAGStatus; import org.apache.tez.dag.api.client.StatusGetOpts; @@ -187,7 +188,7 @@ public class FilterLinesByWord extends Configured implements Tool { FilterByWordInputProcessor.class.getName()).setUserPayload(stage1Payload), stage1NumTasks, MRHelpers.getMapResource(stage1Conf)); if (generateSplitsInClient) { - stage1Vertex.setTaskLocationsHint(inputSplitInfo.getTaskLocationHints()); + stage1Vertex.setLocationHint(new VertexLocationHint(inputSplitInfo.getTaskLocationHints())); Map stage1LocalResources = new HashMap(); stage1LocalResources.putAll(commonLocalResources); MRHelpers.updateLocalResourcesForInputSplits(fs, inputSplitInfo, stage1LocalResources); http://git-wip-us.apache.org/repos/asf/tez/blob/0df3d0f6/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWordOneToOne.java ---------------------------------------------------------------------- diff --git a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWordOneToOne.java b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWordOneToOne.java index 9143351..83be555 100644 --- a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWordOneToOne.java +++ b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWordOneToOne.java @@ -56,6 +56,7 @@ import org.apache.tez.dag.api.TezConfiguration; import org.apache.tez.dag.api.TezException; import org.apache.tez.dag.api.TezUncheckedException; import org.apache.tez.dag.api.Vertex; +import org.apache.tez.dag.api.VertexLocationHint; import org.apache.tez.dag.api.client.DAGClient; import org.apache.tez.dag.api.client.DAGStatus; import org.apache.tez.mapreduce.committer.MROutputCommitter; @@ -173,7 +174,7 @@ public class FilterLinesByWordOneToOne extends Configured implements Tool { FilterByWordInputProcessor.class.getName()).setUserPayload(stage1Payload), stage1NumTasks, MRHelpers.getMapResource(stage1Conf)); if (generateSplitsInClient) { - stage1Vertex.setTaskLocationsHint(inputSplitInfo.getTaskLocationHints()); + stage1Vertex.setLocationHint(new VertexLocationHint(inputSplitInfo.getTaskLocationHints())); Map stage1LocalResources = new HashMap(); stage1LocalResources.putAll(commonLocalResources); MRHelpers.updateLocalResourcesForInputSplits(fs, inputSplitInfo, stage1LocalResources); http://git-wip-us.apache.org/repos/asf/tez/blob/0df3d0f6/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/MRRSleepJob.java ---------------------------------------------------------------------- diff --git a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/MRRSleepJob.java b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/MRRSleepJob.java index ec49a1c..ab75441 100644 --- a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/MRRSleepJob.java +++ b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/MRRSleepJob.java @@ -69,6 +69,7 @@ import org.apache.tez.dag.api.ProcessorDescriptor; import org.apache.tez.dag.api.TezConfiguration; import org.apache.tez.dag.api.TezUncheckedException; import org.apache.tez.dag.api.Vertex; +import org.apache.tez.dag.api.VertexLocationHint; import org.apache.tez.dag.api.client.DAGClient; import org.apache.tez.dag.api.client.DAGStatus; import org.apache.tez.mapreduce.common.MRInputAMSplitGenerator; @@ -546,7 +547,7 @@ public class MRRSleepJob extends Configured implements Tool { MapProcessor.class.getName()).setUserPayload(mapUserPayload), numTasks, MRHelpers.getMapResource(mapStageConf)); if (!generateSplitsInAM) { - mapVertex.setTaskLocationsHint(inputSplitInfo.getTaskLocationHints()); + mapVertex.setLocationHint(new VertexLocationHint(inputSplitInfo.getTaskLocationHints())); } if (writeSplitsToDFS) { http://git-wip-us.apache.org/repos/asf/tez/blob/0df3d0f6/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/OrderedWordCount.java ---------------------------------------------------------------------- diff --git a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/OrderedWordCount.java b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/OrderedWordCount.java index 7af5402..14240ea 100644 --- a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/OrderedWordCount.java +++ b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/OrderedWordCount.java @@ -216,7 +216,7 @@ public class OrderedWordCount extends Configured implements Tool { .setHistoryText(mapStageHistoryText), numMaps, MRHelpers.getMapResource(mapStageConf)); if (generateSplitsInClient) { - mapVertex.setTaskLocationsHint(inputSplitInfo.getTaskLocationHints()); + mapVertex.setLocationHint(new VertexLocationHint(inputSplitInfo.getTaskLocationHints())); Map mapLocalResources = new HashMap(); mapLocalResources.putAll(commonLocalResources); http://git-wip-us.apache.org/repos/asf/tez/blob/0df3d0f6/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/YARNRunner.java ---------------------------------------------------------------------- diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/YARNRunner.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/YARNRunner.java index d785be3..2bc2ff0 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/YARNRunner.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/YARNRunner.java @@ -84,6 +84,7 @@ import org.apache.tez.dag.api.ProcessorDescriptor; import org.apache.tez.dag.api.TezConfiguration; import org.apache.tez.dag.api.TezException; import org.apache.tez.dag.api.Vertex; +import org.apache.tez.dag.api.VertexLocationHint; import org.apache.tez.dag.api.VertexLocationHint.TaskLocationHint; import org.apache.tez.dag.api.VertexManagerPluginDescriptor; import org.apache.tez.dag.api.client.DAGStatus; @@ -424,7 +425,7 @@ public class YARNRunner implements ClientProtocol { vertex.setTaskEnvironment(taskEnv) .setTaskLocalFiles(taskLocalResources) - .setTaskLocationsHint(locations) + .setLocationHint(new VertexLocationHint(locations)) .setTaskLaunchCmdOpts(taskJavaOpts); if (!isMap) { http://git-wip-us.apache.org/repos/asf/tez/blob/0df3d0f6/tez-tests/src/test/java/org/apache/tez/mapreduce/TestMRRJobsDAGApi.java ---------------------------------------------------------------------- diff --git a/tez-tests/src/test/java/org/apache/tez/mapreduce/TestMRRJobsDAGApi.java b/tez-tests/src/test/java/org/apache/tez/mapreduce/TestMRRJobsDAGApi.java index b0d5061..44f1343 100644 --- a/tez-tests/src/test/java/org/apache/tez/mapreduce/TestMRRJobsDAGApi.java +++ b/tez-tests/src/test/java/org/apache/tez/mapreduce/TestMRRJobsDAGApi.java @@ -82,6 +82,7 @@ import org.apache.tez.dag.api.TezConfiguration; import org.apache.tez.dag.api.TezException; import org.apache.tez.dag.api.TezUncheckedException; import org.apache.tez.dag.api.Vertex; +import org.apache.tez.dag.api.VertexLocationHint; import org.apache.tez.dag.api.client.DAGClient; import org.apache.tez.dag.api.client.DAGStatus; import org.apache.tez.dag.api.client.DAGStatus.State; @@ -497,7 +498,7 @@ public class TestMRRJobsDAGApi { LocalResourceType.FILE, LocalResourceVisibility.APPLICATION)); stage1Vertex.setTaskLocalFiles(stage1LocalResources); - stage1Vertex.setTaskLocationsHint(inputSplitInfo.getTaskLocationHints()); + stage1Vertex.setLocationHint(new VertexLocationHint(inputSplitInfo.getTaskLocationHints())); } // TODO env, resources