tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From rbalamo...@apache.org
Subject [1/2] git commit: TEZ-1614. Use setFromConfiguration() in SortMergeJoinExample to demonstrate the usage (Cherry picked from commit efa6afd0f3e8a5c2cde8f1c415f6b3f87c194c89)
Date Thu, 09 Oct 2014 22:57:29 GMT
Repository: tez
Updated Branches:
  refs/heads/branch-0.5 b48bc6ab9 -> eea4cea4e


TEZ-1614.  Use setFromConfiguration() in SortMergeJoinExample to demonstrate the usage
 (Cherry picked from commit efa6afd0f3e8a5c2cde8f1c415f6b3f87c194c89)


Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/c05c4649
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/c05c4649
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/c05c4649

Branch: refs/heads/branch-0.5
Commit: c05c4649cee95288704672b1944a4b7398bdf1a8
Parents: b48bc6a
Author: Rajesh Balamohan <rbalamohan@apache.org>
Authored: Fri Oct 10 04:12:13 2014 +0530
Committer: Rajesh Balamohan <rbalamohan@apache.org>
Committed: Fri Oct 10 04:22:48 2014 +0530

----------------------------------------------------------------------
 .../main/java/org/apache/tez/examples/HashJoinExample.java    | 4 ++--
 .../src/main/java/org/apache/tez/examples/JoinValidate.java   | 2 +-
 .../main/java/org/apache/tez/examples/OrderedWordCount.java   | 4 ++--
 .../java/org/apache/tez/examples/SortMergeJoinExample.java    | 2 ++
 .../src/main/java/org/apache/tez/examples/WordCount.java      | 2 +-
 .../runtime/library/conf/OrderedPartitionedKVEdgeConfig.java  | 7 +++++++
 .../tez/runtime/library/conf/UnorderedKVEdgeConfig.java       | 7 +++++++
 .../library/conf/UnorderedPartitionedKVEdgeConfig.java        | 7 +++++++
 8 files changed, 29 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tez/blob/c05c4649/tez-examples/src/main/java/org/apache/tez/examples/HashJoinExample.java
----------------------------------------------------------------------
diff --git a/tez-examples/src/main/java/org/apache/tez/examples/HashJoinExample.java b/tez-examples/src/main/java/org/apache/tez/examples/HashJoinExample.java
index 76e53e1..3f9085f 100644
--- a/tez-examples/src/main/java/org/apache/tez/examples/HashJoinExample.java
+++ b/tez-examples/src/main/java/org/apache/tez/examples/HashJoinExample.java
@@ -266,7 +266,7 @@ public class HashJoinExample extends Configured implements Tool {
     UnorderedPartitionedKVEdgeConfig streamConf =
         UnorderedPartitionedKVEdgeConfig
             .newBuilder(Text.class.getName(), NullWritable.class.getName(),
-                HashPartitioner.class.getName()).setFromConfiguration(tezConf)
+                HashPartitioner.class.getName())
             .build();
 
     /**
@@ -293,7 +293,7 @@ public class HashJoinExample extends Configured implements Tool {
       UnorderedKVEdgeConfig broadcastConf =
           UnorderedKVEdgeConfig
               .newBuilder(Text.class.getName(), NullWritable.class.getName())
-              .setFromConfiguration(tezConf).build();
+              .build();
       hashSideEdgeProperty = broadcastConf.createDefaultBroadcastEdgeProperty();
     } else {
       /**

http://git-wip-us.apache.org/repos/asf/tez/blob/c05c4649/tez-examples/src/main/java/org/apache/tez/examples/JoinValidate.java
----------------------------------------------------------------------
diff --git a/tez-examples/src/main/java/org/apache/tez/examples/JoinValidate.java b/tez-examples/src/main/java/org/apache/tez/examples/JoinValidate.java
index 17e3dc8..0b8671d 100644
--- a/tez-examples/src/main/java/org/apache/tez/examples/JoinValidate.java
+++ b/tez-examples/src/main/java/org/apache/tez/examples/JoinValidate.java
@@ -186,7 +186,7 @@ public class JoinValidate extends Configured implements Tool {
     // better mechanism to configure the IOs.
     OrderedPartitionedKVEdgeConfig edgeConf = OrderedPartitionedKVEdgeConfig
         .newBuilder(Text.class.getName(), NullWritable.class.getName(),
-            HashPartitioner.class.getName()).setFromConfiguration(tezConf).build();
+            HashPartitioner.class.getName()).build();
 
     Vertex lhsVertex = Vertex.create(LHS_INPUT_NAME, ProcessorDescriptor.create(
         ForwardingProcessor.class.getName())).addDataSource("lhs",

http://git-wip-us.apache.org/repos/asf/tez/blob/c05c4649/tez-examples/src/main/java/org/apache/tez/examples/OrderedWordCount.java
----------------------------------------------------------------------
diff --git a/tez-examples/src/main/java/org/apache/tez/examples/OrderedWordCount.java b/tez-examples/src/main/java/org/apache/tez/examples/OrderedWordCount.java
index 48884f4..fd18592 100644
--- a/tez-examples/src/main/java/org/apache/tez/examples/OrderedWordCount.java
+++ b/tez-examples/src/main/java/org/apache/tez/examples/OrderedWordCount.java
@@ -140,7 +140,7 @@ public class OrderedWordCount extends Configured implements Tool  {
     // Use Text key and IntWritable value to bring counts for each word in the same partition
     OrderedPartitionedKVEdgeConfig summationEdgeConf = OrderedPartitionedKVEdgeConfig
         .newBuilder(Text.class.getName(), IntWritable.class.getName(),
-            HashPartitioner.class.getName()).setFromConfiguration(tezConf).build();
+            HashPartitioner.class.getName()).build();
 
     // This vertex will be reading intermediate data via an input edge and writing intermediate
data
     // via an output edge.
@@ -151,7 +151,7 @@ public class OrderedWordCount extends Configured implements Tool  {
     // partition. The data will be ordered by count and words grouped by count.
     OrderedPartitionedKVEdgeConfig sorterEdgeConf = OrderedPartitionedKVEdgeConfig
         .newBuilder(IntWritable.class.getName(), Text.class.getName(),
-            HashPartitioner.class.getName()).setFromConfiguration(tezConf).build();
+            HashPartitioner.class.getName()).build();
 
     // Use 1 task to bring all the data in one place for global sorted order. Essentially
the number
     // of partitions is 1. So the NoOpSorter can be used to produce the globally ordered
output

http://git-wip-us.apache.org/repos/asf/tez/blob/c05c4649/tez-examples/src/main/java/org/apache/tez/examples/SortMergeJoinExample.java
----------------------------------------------------------------------
diff --git a/tez-examples/src/main/java/org/apache/tez/examples/SortMergeJoinExample.java
b/tez-examples/src/main/java/org/apache/tez/examples/SortMergeJoinExample.java
index b9ba87a..25d2a24 100644
--- a/tez-examples/src/main/java/org/apache/tez/examples/SortMergeJoinExample.java
+++ b/tez-examples/src/main/java/org/apache/tez/examples/SortMergeJoinExample.java
@@ -266,6 +266,8 @@ public class SortMergeJoinExample extends Configured implements Tool {
      * JoinProcessor. The number of fragments is initially inferred from the
      * number of tasks running in the join vertex because each task will be
      * handling one fragment.
+     * Edge config options are derived from client-side tez-site.xml (recommended). Optionally
+     * invoke setFromConfiguration to override these config options via commandline arguments.
      */
     OrderedPartitionedKVEdgeConfig edgeConf =
         OrderedPartitionedKVEdgeConfig

http://git-wip-us.apache.org/repos/asf/tez/blob/c05c4649/tez-examples/src/main/java/org/apache/tez/examples/WordCount.java
----------------------------------------------------------------------
diff --git a/tez-examples/src/main/java/org/apache/tez/examples/WordCount.java b/tez-examples/src/main/java/org/apache/tez/examples/WordCount.java
index 89eb68c..e0e94cb 100644
--- a/tez-examples/src/main/java/org/apache/tez/examples/WordCount.java
+++ b/tez-examples/src/main/java/org/apache/tez/examples/WordCount.java
@@ -173,7 +173,7 @@ public class WordCount extends Configured implements Tool {
     // using this helper.
     OrderedPartitionedKVEdgeConfig edgeConf = OrderedPartitionedKVEdgeConfig
         .newBuilder(Text.class.getName(), IntWritable.class.getName(),
-            HashPartitioner.class.getName()).setFromConfiguration(tezConf).build();
+            HashPartitioner.class.getName()).build();
 
     // Create a vertex that reads the tokenized data and calculates the sum using the SumProcessor.
     // The number of tasks that do the work of this vertex depends on the number of partitions
used 

http://git-wip-us.apache.org/repos/asf/tez/blob/c05c4649/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OrderedPartitionedKVEdgeConfig.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OrderedPartitionedKVEdgeConfig.java
b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OrderedPartitionedKVEdgeConfig.java
index 76f73ac..0f10cf1 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OrderedPartitionedKVEdgeConfig.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OrderedPartitionedKVEdgeConfig.java
@@ -259,6 +259,13 @@ public class OrderedPartitionedKVEdgeConfig
     }
 
     @Override
+    /**
+     * Edge config options are derived from client-side tez-site.xml (recommended).
+     * Optionally invoke setFromConfiguration to override these config options via commandline
arguments.
+     *
+     * @param conf
+     * @return this object for further chained method calls
+     */
     public Builder setFromConfiguration(Configuration conf) {
       outputBuilder.setFromConfiguration(conf);
       inputBuilder.setFromConfiguration(conf);

http://git-wip-us.apache.org/repos/asf/tez/blob/c05c4649/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedKVEdgeConfig.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedKVEdgeConfig.java
b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedKVEdgeConfig.java
index 3864565..6eb1d6a 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedKVEdgeConfig.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedKVEdgeConfig.java
@@ -193,6 +193,13 @@ public class UnorderedKVEdgeConfig extends HadoopKeyValuesBasedBaseEdgeConfig
{
     }
 
     @Override
+    /**
+     * Edge config options are derived from client-side tez-site.xml (recommended).
+     * Optionally invoke setFromConfiguration to override these config options via commandline
arguments.
+     *
+     * @param conf
+     * @return this object for further chained method calls
+     */
     public Builder setFromConfiguration(Configuration conf) {
       outputBuilder.setFromConfiguration(conf);
       inputBuilder.setFromConfiguration(conf);

http://git-wip-us.apache.org/repos/asf/tez/blob/c05c4649/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedPartitionedKVEdgeConfig.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedPartitionedKVEdgeConfig.java
b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedPartitionedKVEdgeConfig.java
index 44ca74e..30585bd 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedPartitionedKVEdgeConfig.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedPartitionedKVEdgeConfig.java
@@ -199,6 +199,13 @@ public class UnorderedPartitionedKVEdgeConfig
     }
 
     @Override
+    /**
+     * Edge config options are derived from client-side tez-site.xml (recommended).
+     * Optionally invoke setFromConfiguration to override these config options via commandline
arguments.
+     *
+     * @param conf
+     * @return this object for further chained method calls
+     */
     public Builder setFromConfiguration(Configuration conf) {
       outputBuilder.setFromConfiguration(conf);
       inputBuilder.setFromConfiguration(conf);


Mime
View raw message