tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hit...@apache.org
Subject git commit: TEZ-1575. MRRSleepJob does not pick MR settings for container size and java opts. (hitesh)
Date Fri, 12 Sep 2014 18:03:16 GMT
Repository: tez
Updated Branches:
  refs/heads/master d1ea44d0b -> b62298e10


TEZ-1575. MRRSleepJob does not pick MR settings for container size and java opts. (hitesh)


Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/b62298e1
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/b62298e1
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/b62298e1

Branch: refs/heads/master
Commit: b62298e1054c934e4ee8909b7ba3165ce344ea8b
Parents: d1ea44d
Author: Hitesh Shah <hitesh@apache.org>
Authored: Fri Sep 12 11:02:51 2014 -0700
Committer: Hitesh Shah <hitesh@apache.org>
Committed: Fri Sep 12 11:02:51 2014 -0700

----------------------------------------------------------------------
 .../tez/mapreduce/examples/MRRSleepJob.java     | 25 +++++++++++++-------
 1 file changed, 16 insertions(+), 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tez/blob/b62298e1/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/MRRSleepJob.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/MRRSleepJob.java b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/MRRSleepJob.java
index 3f9d222..de8a1ad 100644
--- a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/MRRSleepJob.java
+++ b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/MRRSleepJob.java
@@ -525,9 +525,11 @@ public class MRRSleepJob extends Configured implements Tool {
     int numTasks = generateSplitsInAM ? -1 : numMapper;
 
     Vertex mapVertex = Vertex.create("map", ProcessorDescriptor.create(
-        MapProcessor.class.getName()).setUserPayload(mapUserPayload), numTasks)
-        .addTaskLocalFiles(commonLocalResources);
-    mapVertex.addDataSource("MRInput", dataSource);
+        MapProcessor.class.getName()).setUserPayload(mapUserPayload), numTasks,
+        MRHelpers.getResourceForMRMapper(mapStageConf));
+    mapVertex.addTaskLocalFiles(commonLocalResources)
+        .addDataSource("MRInput", dataSource)
+        .setTaskLaunchCmdOpts(MRHelpers.getJavaOptsForMRMapper(mapStageConf));
     vertices.add(mapVertex);
 
     if (iReduceStagesCount > 0
@@ -538,8 +540,11 @@ public class MRRSleepJob extends Configured implements Tool {
         UserPayload iReduceUserPayload = TezUtils.createUserPayloadFromConf(iconf);
         Vertex ivertex = Vertex.create("ireduce" + (i + 1),
             ProcessorDescriptor.create(ReduceProcessor.class.getName()).
-                setUserPayload(iReduceUserPayload), numIReducer);
-        ivertex.addTaskLocalFiles(commonLocalResources);
+                setUserPayload(iReduceUserPayload), numIReducer,
+            MRHelpers.getResourceForMRReducer(intermediateReduceStageConfs[i]));
+        ivertex.addTaskLocalFiles(commonLocalResources)
+            .setTaskLaunchCmdOpts(MRHelpers.getJavaOptsForMRReducer(
+                intermediateReduceStageConfs[i]));
         vertices.add(ivertex);
       }
     }
@@ -548,10 +553,12 @@ public class MRRSleepJob extends Configured implements Tool {
     if (numReducer > 0) {
       UserPayload reducePayload = TezUtils.createUserPayloadFromConf(finalReduceConf);
       finalReduceVertex = Vertex.create("reduce", ProcessorDescriptor.create(
-          ReduceProcessor.class.getName()).setUserPayload(reducePayload), numReducer);
-      finalReduceVertex.addTaskLocalFiles(commonLocalResources);
-      finalReduceVertex.addDataSink("MROutput", MROutputLegacy.createConfigBuilder(finalReduceConf,
-          NullOutputFormat.class).build());
+          ReduceProcessor.class.getName()).setUserPayload(reducePayload), numReducer,
+          MRHelpers.getResourceForMRReducer(finalReduceConf));
+      finalReduceVertex.addTaskLocalFiles(commonLocalResources)
+          .addDataSink("MROutput", MROutputLegacy.createConfigBuilder(
+              finalReduceConf, NullOutputFormat.class).build())
+          .setTaskLaunchCmdOpts(MRHelpers.getJavaOptsForMRReducer(finalReduceConf));
       vertices.add(finalReduceVertex);
     } else {
       // Map only job


Mime
View raw message