tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ss...@apache.org
Subject git commit: TEZ-826. Remove wordcountmrrtest example. (sseth)
Date Thu, 13 Feb 2014 07:52:55 GMT
Updated Branches:
  refs/heads/master 3a63d9b1a -> 787921e80


TEZ-826. Remove wordcountmrrtest example. (sseth)


Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/787921e8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/787921e8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/787921e8

Branch: refs/heads/master
Commit: 787921e801710d6bfe1ca076c0ecd286e0527704
Parents: 3a63d9b
Author: Siddharth Seth <sseth@apache.org>
Authored: Wed Feb 12 23:52:00 2014 -0800
Committer: Siddharth Seth <sseth@apache.org>
Committed: Wed Feb 12 23:52:00 2014 -0800

----------------------------------------------------------------------
 .../tez/mapreduce/examples/ExampleDriver.java   |   3 -
 .../mapreduce/examples/WordCountMRRTest.java    | 128 -------------------
 2 files changed, 131 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/787921e8/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/ExampleDriver.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/ExampleDriver.java
b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/ExampleDriver.java
index d07db3c..34570b5 100644
--- a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/ExampleDriver.java
+++ b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/ExampleDriver.java
@@ -52,9 +52,6 @@ public class ExampleDriver {
       pgd.addClass("mapredwordcount", MapredWordCount.class,
           "A map/reduce program that counts the words in the input files"
          + " using the mapred apis.");
-      pgd.addClass("wordcountmrrtest", WordCountMRRTest.class,
-          "A map/reduce program that counts the words in the input files."
-          + " Map splits on spaces. First reduce splits on \".\"");
       pgd.addClass("randomwriter", RandomWriter.class,
           "A map/reduce program that writes 10GB of random data per node.");
       pgd.addClass("randomtextwriter", RandomTextWriter.class,

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/787921e8/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/WordCountMRRTest.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/WordCountMRRTest.java
b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/WordCountMRRTest.java
deleted file mode 100644
index aa9c150..0000000
--- a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/WordCountMRRTest.java
+++ /dev/null
@@ -1,128 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tez.mapreduce.examples;
-
-import java.io.IOException;
-import java.util.StringTokenizer;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.Mapper;
-import org.apache.hadoop.mapreduce.Reducer;
-import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
-import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
-import org.apache.hadoop.util.GenericOptionsParser;
-import org.apache.tez.mapreduce.hadoop.MRJobConfig;
-import org.apache.tez.mapreduce.hadoop.MultiStageMRConfigUtil;
-
-public class WordCountMRRTest {
-  
-
-  public static class TokenizerMapper 
-       extends Mapper<Object, Text, Text, IntWritable>{
-    
-    private final static IntWritable one = new IntWritable(1);
-    private Text word = new Text();
-      
-    public void map(Object key, Text value, Context context
-                    ) throws IOException, InterruptedException {
-      StringTokenizer itr = new StringTokenizer(value.toString());
-      while (itr.hasMoreTokens()) {
-        word.set(itr.nextToken());
-        context.write(word, one);
-      }
-    }
-  }
-
-  public static class IntermediateReducer extends
-      Reducer<Text, IntWritable, Text, IntWritable> {
-    private IntWritable result = new IntWritable();
-
-    public void reduce(Text key, Iterable<IntWritable> values, Context context)
-        throws IOException, InterruptedException {
-
-      String keyString = key.toString();
-
-      int sum = 0;
-      for (IntWritable val : values) {
-        sum += val.get();
-      }
-      result.set(sum);
-      String[] splits = keyString.split("\\.");
-      for (String split : splits) {
-        context.write(new Text(split), result);
-      }
-
-    }
-  }
-
-  public static class IntSumReducer 
-       extends Reducer<Text,IntWritable,Text,IntWritable> {
-    private IntWritable result = new IntWritable();
-
-    public void reduce(Text key, Iterable<IntWritable> values, 
-                       Context context
-                       ) throws IOException, InterruptedException {
-      int sum = 0;
-      for (IntWritable val : values) {
-        sum += val.get();
-      }
-      result.set(sum);
-      context.write(key, result);
-    }
-  }
-
-  public static void main(String[] args) throws Exception {
-    Configuration conf = new Configuration();
-    
-    // TODO HACKS This needs to be fixed once the mrr api is detailed.
-    // Has to be set before initialzing job, since it creates a copy.
-    // Alternately use JonConf. 
-    conf.setInt(MRJobConfig.MRR_INTERMEDIATE_STAGES, 1);
-    conf.setClass(MultiStageMRConfigUtil.getPropertyNameForIntermediateStage(1,
-        "mapreduce.job.combine.class"), IntermediateReducer.class,
-        Reducer.class);
-    conf.setClass(MultiStageMRConfigUtil.getPropertyNameForIntermediateStage(1,
-        "mapreduce.job.reduce.class"), IntermediateReducer.class, Reducer.class);
-    conf.set(MultiStageMRConfigUtil.getPropertyNameForIntermediateStage(1,
-        "mapreduce.map.output.value.class"), IntWritable.class.getName());
-    
-    
-    
-    System.err.println("Running wordcountMrrTest");
-    String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
-    if (otherArgs.length != 2) {
-      System.err.println("Usage: wordcountmrrtest <in> <out>");
-      System.exit(2);
-    }
-    Job job = new Job(conf, "word count mrr");
-    job.setJarByClass(WordCountMRRTest.class);
-    job.setMapperClass(TokenizerMapper.class);
-    
-    job.setCombinerClass(IntSumReducer.class);
-    job.setReducerClass(IntSumReducer.class);
-    job.setOutputKeyClass(Text.class);
-    job.setOutputValueClass(IntWritable.class);
-    FileInputFormat.addInputPath(job, new Path(otherArgs[0]));
-    FileOutputFormat.setOutputPath(job, new Path(otherArgs[1]));
-    System.exit(job.waitForCompletion(true) ? 0 : 1);
-  }
-}


Mime
View raw message