tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From rbalamo...@apache.org
Subject git commit: TEZ-1587. Some tez-examples fail in local mode (Contributed by Prakash Ramachandran) (cherry picked from commit e328055e61a8dcaa1beed25e6da47f4a2279f3b6)
Date Wed, 17 Sep 2014 10:27:53 GMT
Repository: tez
Updated Branches:
  refs/heads/branch-0.5 10b1f2058 -> 2d04d64aa


TEZ-1587. Some tez-examples fail in local mode (Contributed by Prakash Ramachandran)
(cherry picked from commit e328055e61a8dcaa1beed25e6da47f4a2279f3b6)


Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/2d04d64a
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/2d04d64a
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/2d04d64a

Branch: refs/heads/branch-0.5
Commit: 2d04d64aa770d7b1cc646bbe7a75c97dfe50afc3
Parents: 10b1f20
Author: Rajesh Balamohan <rbalamohan@apache.org>
Authored: Wed Sep 17 15:54:01 2014 +0530
Committer: Rajesh Balamohan <rbalamohan@apache.org>
Committed: Wed Sep 17 15:57:24 2014 +0530

----------------------------------------------------------------------
 CHANGES.txt                                       |  1 +
 .../java/org/apache/tez/examples/JoinExample.java |  4 ++--
 .../org/apache/tez/examples/JoinValidate.java     |  2 +-
 .../org/apache/tez/examples/OrderedWordCount.java |  4 ++--
 .../java/org/apache/tez/examples/WordCount.java   |  2 +-
 .../examples/BroadcastAndOneToOneExample.java     | 18 ++++++++++++++----
 .../tez/mapreduce/examples/FilterLinesByWord.java |  3 ++-
 .../examples/FilterLinesByWordOneToOne.java       |  3 ++-
 8 files changed, 25 insertions(+), 12 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tez/blob/2d04d64a/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index ebaced8..e90634d 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -23,6 +23,7 @@ ALL CHANGES
   TEZ-1571. Add create method for DataSinkDescriptor.
   TEZ-1585. Memory leak in tez session mode.
   TEZ-1533. Request Events more often if a complete set of events is received by a task.
+  TEZ-1587. Some tez-examples fail in local mode.
 
 Release 0.5.0: 2014-09-03
 

http://git-wip-us.apache.org/repos/asf/tez/blob/2d04d64a/tez-examples/src/main/java/org/apache/tez/examples/JoinExample.java
----------------------------------------------------------------------
diff --git a/tez-examples/src/main/java/org/apache/tez/examples/JoinExample.java b/tez-examples/src/main/java/org/apache/tez/examples/JoinExample.java
index 5ead275..3611fd6 100644
--- a/tez-examples/src/main/java/org/apache/tez/examples/JoinExample.java
+++ b/tez-examples/src/main/java/org/apache/tez/examples/JoinExample.java
@@ -244,7 +244,7 @@ public class JoinExample extends Configured implements Tool {
     UnorderedPartitionedKVEdgeConfig streamConf =
         UnorderedPartitionedKVEdgeConfig
             .newBuilder(Text.class.getName(), NullWritable.class.getName(),
-                HashPartitioner.class.getName()).build();
+                HashPartitioner.class.getName()).setFromConfiguration(tezConf).build();
 
     /**
      * Connect the join vertex with the stream side
@@ -265,7 +265,7 @@ public class JoinExample extends Configured implements Tool {
        * value is null.
        */
       UnorderedKVEdgeConfig broadcastConf = UnorderedKVEdgeConfig.newBuilder(Text.class.getName(),
-          NullWritable.class.getName()).build();
+          NullWritable.class.getName()).setFromConfiguration(tezConf).build();
       hashSideEdgeProperty = broadcastConf.createDefaultBroadcastEdgeProperty();
     } else {
       /**

http://git-wip-us.apache.org/repos/asf/tez/blob/2d04d64a/tez-examples/src/main/java/org/apache/tez/examples/JoinValidate.java
----------------------------------------------------------------------
diff --git a/tez-examples/src/main/java/org/apache/tez/examples/JoinValidate.java b/tez-examples/src/main/java/org/apache/tez/examples/JoinValidate.java
index 3630184..e94df09 100644
--- a/tez-examples/src/main/java/org/apache/tez/examples/JoinValidate.java
+++ b/tez-examples/src/main/java/org/apache/tez/examples/JoinValidate.java
@@ -186,7 +186,7 @@ public class JoinValidate extends Configured implements Tool {
     // better mechanism to configure the IOs.
     OrderedPartitionedKVEdgeConfig edgeConf = OrderedPartitionedKVEdgeConfig
         .newBuilder(Text.class.getName(), NullWritable.class.getName(),
-            HashPartitioner.class.getName()).build();
+            HashPartitioner.class.getName()).setFromConfiguration(tezConf).build();
 
     Vertex lhsVertex = Vertex.create(LHS_INPUT_NAME, ProcessorDescriptor.create(
         ForwardingProcessor.class.getName())).addDataSource("lhs",

http://git-wip-us.apache.org/repos/asf/tez/blob/2d04d64a/tez-examples/src/main/java/org/apache/tez/examples/OrderedWordCount.java
----------------------------------------------------------------------
diff --git a/tez-examples/src/main/java/org/apache/tez/examples/OrderedWordCount.java b/tez-examples/src/main/java/org/apache/tez/examples/OrderedWordCount.java
index fd18592..48884f4 100644
--- a/tez-examples/src/main/java/org/apache/tez/examples/OrderedWordCount.java
+++ b/tez-examples/src/main/java/org/apache/tez/examples/OrderedWordCount.java
@@ -140,7 +140,7 @@ public class OrderedWordCount extends Configured implements Tool  {
     // Use Text key and IntWritable value to bring counts for each word in the same partition
     OrderedPartitionedKVEdgeConfig summationEdgeConf = OrderedPartitionedKVEdgeConfig
         .newBuilder(Text.class.getName(), IntWritable.class.getName(),
-            HashPartitioner.class.getName()).build();
+            HashPartitioner.class.getName()).setFromConfiguration(tezConf).build();
 
     // This vertex will be reading intermediate data via an input edge and writing intermediate
data
     // via an output edge.
@@ -151,7 +151,7 @@ public class OrderedWordCount extends Configured implements Tool  {
     // partition. The data will be ordered by count and words grouped by count.
     OrderedPartitionedKVEdgeConfig sorterEdgeConf = OrderedPartitionedKVEdgeConfig
         .newBuilder(IntWritable.class.getName(), Text.class.getName(),
-            HashPartitioner.class.getName()).build();
+            HashPartitioner.class.getName()).setFromConfiguration(tezConf).build();
 
     // Use 1 task to bring all the data in one place for global sorted order. Essentially
the number
     // of partitions is 1. So the NoOpSorter can be used to produce the globally ordered
output

http://git-wip-us.apache.org/repos/asf/tez/blob/2d04d64a/tez-examples/src/main/java/org/apache/tez/examples/WordCount.java
----------------------------------------------------------------------
diff --git a/tez-examples/src/main/java/org/apache/tez/examples/WordCount.java b/tez-examples/src/main/java/org/apache/tez/examples/WordCount.java
index e0e94cb..89eb68c 100644
--- a/tez-examples/src/main/java/org/apache/tez/examples/WordCount.java
+++ b/tez-examples/src/main/java/org/apache/tez/examples/WordCount.java
@@ -173,7 +173,7 @@ public class WordCount extends Configured implements Tool {
     // using this helper.
     OrderedPartitionedKVEdgeConfig edgeConf = OrderedPartitionedKVEdgeConfig
         .newBuilder(Text.class.getName(), IntWritable.class.getName(),
-            HashPartitioner.class.getName()).build();
+            HashPartitioner.class.getName()).setFromConfiguration(tezConf).build();
 
     // Create a vertex that reads the tokenized data and calculates the sum using the SumProcessor.
     // The number of tasks that do the work of this vertex depends on the number of partitions
used 

http://git-wip-us.apache.org/repos/asf/tez/blob/2d04d64a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/BroadcastAndOneToOneExample.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/BroadcastAndOneToOneExample.java
b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/BroadcastAndOneToOneExample.java
index 81f9581..e8a137c 100644
--- a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/BroadcastAndOneToOneExample.java
+++ b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/BroadcastAndOneToOneExample.java
@@ -50,6 +50,7 @@ import org.apache.tez.runtime.api.ObjectRegistry;
 import org.apache.tez.runtime.api.ProcessorContext;
 import org.apache.tez.runtime.library.api.KeyValueReader;
 import org.apache.tez.runtime.library.api.KeyValueWriter;
+import org.apache.tez.runtime.library.api.TezRuntimeConfiguration;
 import org.apache.tez.runtime.library.conf.UnorderedKVEdgeConfig;
 import org.apache.tez.runtime.library.output.UnorderedKVOutput;
 import org.apache.tez.runtime.library.processor.SimpleProcessor;
@@ -159,7 +160,8 @@ public class BroadcastAndOneToOneExample extends Configured implements
Tool {
         VertexManagerPluginDescriptor.create(InputReadyVertexManager.class.getName()));
 
     UnorderedKVEdgeConfig edgeConf = UnorderedKVEdgeConfig
-        .newBuilder(Text.class.getName(), IntWritable.class.getName()).build();
+        .newBuilder(Text.class.getName(), IntWritable.class.getName())
+        .setFromConfiguration(tezConf).build();
 
     DAG dag = DAG.create("BroadcastAndOneToOneExample");
     dag.addVertex(inputVertex)
@@ -188,9 +190,9 @@ public class BroadcastAndOneToOneExample extends Configured implements
Tool {
 
     // staging dir
     FileSystem fs = FileSystem.get(tezConf);
-    String stagingDirStr = Path.SEPARATOR + "user" + Path.SEPARATOR
-        + user + Path.SEPARATOR+ ".staging" + Path.SEPARATOR
-        + Path.SEPARATOR + Long.toString(System.currentTimeMillis());    
+    String stagingDirStr = tezConf.get(TezConfiguration.TEZ_AM_STAGING_DIR,
+        TezConfiguration.TEZ_AM_STAGING_DIR_DEFAULT) + Path.SEPARATOR +
+        "BroadcastAndOneToOneExample" +  Path.SEPARATOR + Long.toString(System.currentTimeMillis());
     Path stagingDir = new Path(stagingDirStr);
     tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, stagingDirStr);
     stagingDir = fs.makeQualified(stagingDir);
@@ -240,6 +242,14 @@ public class BroadcastAndOneToOneExample extends Configured implements
Tool {
       printUsage();
       throw new TezException("Invalid command line");
     }
+
+    if (doLocalityCheck &&
+        getConf().getBoolean(TezConfiguration.TEZ_LOCAL_MODE,
+            TezConfiguration.TEZ_LOCAL_MODE_DEFAULT)) {
+      System.out.println("locality check is not valid in local mode. skipping");
+      doLocalityCheck = false;
+    }
+
     boolean status = run(getConf(), doLocalityCheck);
     return status ? 0 : 1;
   }

http://git-wip-us.apache.org/repos/asf/tez/blob/2d04d64a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWord.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWord.java
b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWord.java
index 8430c68..f45b3d6 100644
--- a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWord.java
+++ b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWord.java
@@ -199,7 +199,8 @@ public class FilterLinesByWord extends Configured implements Tool {
     stage2Vertex.addDataSink("MROutput", DataSinkDescriptor.create(od, ocd, null));
 
     UnorderedKVEdgeConfig edgeConf = UnorderedKVEdgeConfig
-        .newBuilder(Text.class.getName(), TextLongPair.class.getName()).build();
+        .newBuilder(Text.class.getName(), TextLongPair.class.getName())
+        .setFromConfiguration(tezConf).build();
 
     DAG dag = DAG.create("FilterLinesByWord");
     Edge edge =

http://git-wip-us.apache.org/repos/asf/tez/blob/2d04d64a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWordOneToOne.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWordOneToOne.java
b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWordOneToOne.java
index 515cf70..49cb30b 100644
--- a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWordOneToOne.java
+++ b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWordOneToOne.java
@@ -190,7 +190,8 @@ public class FilterLinesByWordOneToOne extends Configured implements Tool
{
             OutputCommitterDescriptor.create(MROutputCommitter.class.getName()), null));
 
     UnorderedKVEdgeConfig edgeConf = UnorderedKVEdgeConfig
-        .newBuilder(Text.class.getName(), TextLongPair.class.getName()).build();
+        .newBuilder(Text.class.getName(), TextLongPair.class.getName())
+        .setFromConfiguration(tezConf).build();
 
     DAG dag = DAG.create("FilterLinesByWord");
     Edge edge =


Mime
View raw message