hadoop-mapreduce-user mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From Jagat Singh <jagatsi...@gmail.com>
Subject Re: Errors about MRunit
Date Sun, 21 Apr 2013 02:56:19 GMT
Can you please confirm if you are not mixing old and new mapreduce API


On Sun, Apr 21, 2013 at 12:52 PM, 姚吉龙 <geelongyao@gmail.com> wrote:

> Thanks
> But I am confused about the MRunit. How to use MRunit without maven
> I am just following the tutorial:
> https://cwiki.apache.org/confluence/display/MRUNIT/MRUnit+Tutorial
>
> My test code  is below, the maper and reducer is in another project linked
> in my testing project
>
>
>
> import java.io.IOException;
> import java.util.ArrayList;
> import java.util.List;
> import junit.framework.TestCase;
> import org.apache.hadoop.io.IntWritable;
> import org.apache.hadoop.io.LongWritable;
> import org.apache.hadoop.io.Text;
> import org.apache.hadoop.mapred.lib.IdentityMapper;
> import org.apache.hadoop.mapred.Mapper;
> import org.apache.hadoop.mrunit.mapreduce.MapDriver;
> import org.apache.hadoop.mrunit.mapreduce.MapReduceDriver;
> import org.apache.hadoop.mrunit.mapreduce.ReduceDriver;
> import org.apache.hadoop.mrunit.types.Pair;
> import org.junit.Assert;
> import org.junit.Before;
> import org.junit.Test;
>
> public class UnitTest extends TestCase{
>   MapDriver<Object, Text, Text, Text> mapDriver;
>   ReduceDriver<Text, Text, Text, Text> reduceDriver;
>   MapReduceDriver<Object, Text, Text, Text, Text, Text> mapReduceDriver;
>    @Before
>   public void setUp() {
> carTest.FirstMapper mapper = new carTest.FirstMapper();
>  carTest.FirstReducer reducer = new carTest.FirstReducer();
>     mapDriver = MapDriver.newMapDriver(mapper);;
>     reduceDriver = ReduceDriver.newReduceDriver(reducer);
>     mapReduceDriver = MapReduceDriver.newMapReduceDriver(mapper, reducer);
>   }
>   @Test
>   public void testMapper() throws IOException {
>     mapDriver.withInput(new Object(), new Text(
>
>  "199397,32100000000000000800000120110131014195,321000000000000008,20110131170958,00,,苏K16423,2,,34.00,,,,460,,,K33,02,,1710048437180058,476096"));
>     mapDriver.withOutput(new Text("苏K16423"), new
> Text("321000000000000008"));
>     mapDriver.runTest();
>
>  //"199397","32100000000000000800000120110131014195","321000000000000008","20110131170958","00","","苏K16423","2","","34.00","","","","460","","","K33","02","","1710048437180058","476096"
>
>   }
>   @Test
>   public void testReducer() throws IOException {
>     List<Text> values = new ArrayList<Text>();
>     values.add(new
> Text("199397,32100000000000000800000120110131014195,321000000000000005,20110131172000,00,,苏K16423,2,,34.00,,,,460,,,K33,02,,1710048437180058,476096"));
>   //  values.add(new Text("1"));
>     reduceDriver.withInput(new Text("6"), values);
>     reduceDriver.withOutput(new Text("6"), new Text("2"));
>     reduceDriver.runTest();
>   }
>
>
>   }
>
> While, always failed to pass
> [image: 内嵌图片 1]
> Here is the trace
>
> java.lang.IncompatibleClassChangeError: Found class
> org.apache.hadoop.mapreduce.TaskInputOutputContext, but interface was
> expected
> at
> org.apache.hadoop.mrunit.mapreduce.mock.MockContextWrapper.createCommon(MockContextWrapper.java:53)
>  at
> org.apache.hadoop.mrunit.mapreduce.mock.MockMapContextWrapper.create(MockMapContextWrapper.java:70)
> at
> org.apache.hadoop.mrunit.mapreduce.mock.MockMapContextWrapper.<init>(MockMapContextWrapper.java:62)
>  at org.apache.hadoop.mrunit.mapreduce.MapDriver.run(MapDriver.java:217)
> at org.apache.hadoop.mrunit.MapDriverBase.runTest(MapDriverBase.java:150)
>  at org.apache.hadoop.mrunit.TestDriver.runTest(TestDriver.java:137)
> at UnitTest.testMapper(UnitTest.java:37)
>  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
>  at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
> at java.lang.reflect.Method.invoke(Method.java:597)
>  at junit.framework.TestCase.runTest(TestCase.java:168)
> at junit.framework.TestCase.runBare(TestCase.java:134)
>  at junit.framework.TestResult$1.protect(TestResult.java:110)
> at junit.framework.TestResult.runProtected(TestResult.java:128)
>  at junit.framework.TestResult.run(TestResult.java:113)
> at junit.framework.TestCase.run(TestCase.java:124)
>  at junit.framework.TestSuite.runTest(TestSuite.java:232)
> at junit.framework.TestSuite.run(TestSuite.java:227)
>  at
> org.junit.internal.runners.JUnit38ClassRunner.run(JUnit38ClassRunner.java:79)
> at
> org.eclipse.jdt.internal.junit4.runner.JUnit4TestReference.run(JUnit4TestReference.java:50)
>  at
> org.eclipse.jdt.internal.junit.runner.TestExecution.run(TestExecution.java:38)
> at
> org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.runTests(RemoteTestRunner.java:467)
>  at
> org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.runTests(RemoteTestRunner.java:683)
> at
> org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.run(RemoteTestRunner.java:390)
>  at
> org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.main(RemoteTestRunner.java:197)
>
> Any body help to figure out these?
> I am going crazy
>
>
> BRs
> Geelong
>
>
>
> 2013/4/21 Rishi Yadav <rishi@infoobjects.com>
>
>> Maven is not necessary to use hadoop or MRUnit. That being said I am a
>> big fan of maven. Please find attached wordcount with maven.
>>
>>
>>
>> On Sat, Apr 20, 2013 at 7:17 PM, 姚吉龙 <geelongyao@gmail.com> wrote:
>>
>>> Thank for your reply
>>> But I think the main problem is that I do konw how to use maven with
>>> hadoop or MRunit
>>> Any body can give me a example for MRunit project
>>>
>>>
>>>
>>> 2013/4/21 Rishi Yadav <rishi@infoobjects.com>
>>>
>>>> your problem is simple, you are mixing mapred (old api) and
>>>> mapreduce(new api) libraries. MRUnit has implementation for both apis.
>>>>
>>>> Here's an example of WordCountTest with use of new api.
>>>>
>>>>
>>>> ----------------------------------------------------------------------------------------------------------------------
>>>>
>>>> package com.infoobjects.hadoop.wc;
>>>>
>>>>
>>>> import java.util.ArrayList;
>>>>
>>>> import java.util.List;
>>>>
>>>>
>>>> import org.apache.hadoop.io.IntWritable;
>>>>
>>>> import org.apache.hadoop.io.LongWritable;
>>>>
>>>> import org.apache.hadoop.io.Text;
>>>>
>>>> import org.apache.hadoop.mrunit.mapreduce.MapDriver;
>>>>
>>>> import org.apache.hadoop.mrunit.mapreduce.MapReduceDriver;
>>>>
>>>> import org.apache.hadoop.mrunit.mapreduce.ReduceDriver;
>>>>
>>>> import org.junit.Before;
>>>>
>>>> import org.junit.Test;
>>>>
>>>>
>>>> public class WordCountTest {
>>>>
>>>> MapDriver<LongWritable, Text, Text, IntWritable> mapDriver;
>>>>
>>>> ReduceDriver<Text, IntWritable, Text, IntWritable> reduceDriver;
>>>>
>>>> MapReduceDriver<LongWritable, Text, Text, IntWritable, Text,
>>>> IntWritable> mapReduceDriver;
>>>>
>>>>
>>>>  @Before
>>>>
>>>> public void init() {
>>>>
>>>>  WordMapper mapper = new WordMapper();
>>>>
>>>>  WordReducer reducer = new WordReducer();
>>>>
>>>>  mapDriver = new MapDriver<LongWritable, Text, Text, IntWritable>();
>>>>
>>>>  mapDriver.setMapper(mapper);
>>>>
>>>>     reduceDriver = ReduceDriver.newReduceDriver(reducer);
>>>>
>>>>     mapReduceDriver = MapReduceDriver.newMapReduceDriver(mapper,
>>>> reducer);
>>>>
>>>>  }
>>>>
>>>>
>>>>  @Test
>>>>
>>>> public void testMapper() {
>>>>
>>>>  mapDriver.withInput(new LongWritable(1), new Text("foo bar"));
>>>>
>>>>  mapDriver.withOutput(new Text("foo"), new IntWritable(1));
>>>>
>>>>  mapDriver.withOutput(new Text("bar"), new IntWritable(1));
>>>>
>>>>  mapDriver.runTest();
>>>>
>>>> }
>>>>
>>>>     @Test
>>>>
>>>>   public void testReducer() {
>>>>
>>>>     List<IntWritable> values = new ArrayList<IntWritable>();
>>>>
>>>>     values.add(new IntWritable(1));
>>>>
>>>>     values.add(new IntWritable(1));
>>>>
>>>>     reduceDriver.withInput(new Text("foo"), values);
>>>>
>>>>     reduceDriver.withOutput(new Text("foo"), new IntWritable(2));
>>>>
>>>>     reduceDriver.runTest();
>>>>
>>>>   }
>>>>
>>>>
>>>>
>>>>   @Test
>>>>
>>>>   public void testMapReduce() {
>>>>
>>>>    mapReduceDriver.withInput(new LongWritable(1), new Text("brian
>>>> felix"));
>>>>
>>>>    mapReduceDriver.withOutput(new Text("foo"), new IntWritable(1));
>>>>
>>>>    mapReduceDriver.withOutput(new Text("bar"), new IntWritable(1));
>>>>
>>>>    mapReduceDriver.runTest();
>>>>
>>>>   }
>>>>
>>>>
>>>> }
>>>>
>>>> Thanks and Regards,
>>>>
>>>> Rishi Yadav
>>>>
>>>> (o) 408.988.2000x113 ||  (f) 408.716.2726
>>>>
>>>> InfoObjects Inc || http://www.infoobjects.com *(Big Data Solutions)*
>>>>
>>>> *INC 500 Fastest growing company in 2012 || 2011*
>>>>
>>>> *Best Place to work in Bay Area 2012 - *SF Business Times and the
>>>> Silicon Valley / San Jose Business Journal
>>>>
>>>> 2041 Mission College Boulevard, #280 || Santa Clara, CA 95054
>>>>
>>>>
>>>>
>>>>
>>>> On Sat, Apr 20, 2013 at 7:14 AM, 姚吉龙 <geelongyao@gmail.com> wrote:
>>>>
>>>>> This is what I got form my eclipse. Why still errors about the lib
>>>>> from hadoop
>>>>> [image: 内嵌图片 1][image: 内嵌图片 2]
>>>>> anybody tell me how to use MRunit and Maven
>>>>>
>>>>>
>>>>> 2013/4/20 Hemanth Yamijala <yhemanth@thoughtworks.com>
>>>>>
>>>>>> Hi,
>>>>>>
>>>>>> If your goal is to use the new API, I am able to get it to work with
>>>>>> the following maven configuration:
>>>>>>
>>>>>>     <dependency>
>>>>>>       <groupId>org.apache.mrunit</groupId>
>>>>>>       <artifactId>mrunit</artifactId>
>>>>>>       <version>0.9.0-incubating</version>
>>>>>>       <classifier>hadoop1</classifier>
>>>>>>     </dependency>
>>>>>>
>>>>>> If I switch with classifier hadoop2, I get the same errors as what
>>>>>> you facing.
>>>>>>
>>>>>> Thanks
>>>>>> Hemanth
>>>>>>
>>>>>>
>>>>>> On Sat, Apr 20, 2013 at 3:42 PM, 姚吉龙 <geelongyao@gmail.com>
wrote:
>>>>>>
>>>>>>> Hi Everyone
>>>>>>>
>>>>>>> I am testing my MR programe with MRunit, it's version
>>>>>>> is mrunit-0.9.0-incubating-hadoop2. My hadoop version is 1.0.4
>>>>>>> The error trace is below:
>>>>>>>
>>>>>>> java.lang.IncompatibleClassChangeError: Found class
>>>>>>> org.apache.hadoop.mapreduce.TaskInputOutputContext, but interface
was
>>>>>>> expected
>>>>>>> at
>>>>>>> org.apache.hadoop.mrunit.mapreduce.mock.MockContextWrapper.createCommon(MockContextWrapper.java:53)
>>>>>>>  at
>>>>>>> org.apache.hadoop.mrunit.mapreduce.mock.MockMapContextWrapper.create(MockMapContextWrapper.java:70)
>>>>>>> at
>>>>>>> org.apache.hadoop.mrunit.mapreduce.mock.MockMapContextWrapper.<init>(MockMapContextWrapper.java:62)
>>>>>>>  at
>>>>>>> org.apache.hadoop.mrunit.mapreduce.MapDriver.run(MapDriver.java:217)
>>>>>>> at
>>>>>>> org.apache.hadoop.mrunit.MapDriverBase.runTest(MapDriverBase.java:150)
>>>>>>>  at org.apache.hadoop.mrunit.TestDriver.runTest(TestDriver.java:137)
>>>>>>> at UnitTest.testMapper(UnitTest.java:41)
>>>>>>>  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>>>>>>> at
>>>>>>> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
>>>>>>>  at
>>>>>>> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
>>>>>>> at java.lang.reflect.Method.invoke(Method.java:597)
>>>>>>>  at junit.framework.TestCase.runTest(TestCase.java:168)
>>>>>>> at junit.framework.TestCase.runBare(TestCase.java:134)
>>>>>>>  at junit.framework.TestResult$1.protect(TestResult.java:110)
>>>>>>> at junit.framework.TestResult.runProtected(TestResult.java:128)
>>>>>>>  at junit.framework.TestResult.run(TestResult.java:113)
>>>>>>> at junit.framework.TestCase.run(TestCase.java:124)
>>>>>>>  at junit.framework.TestSuite.runTest(TestSuite.java:232)
>>>>>>> at junit.framework.TestSuite.run(TestSuite.java:227)
>>>>>>>  at
>>>>>>> org.junit.internal.runners.JUnit38ClassRunner.run(JUnit38ClassRunner.java:79)
>>>>>>> at
>>>>>>> org.eclipse.jdt.internal.junit4.runner.JUnit4TestReference.run(JUnit4TestReference.java:50)
>>>>>>>  at
>>>>>>> org.eclipse.jdt.internal.junit.runner.TestExecution.run(TestExecution.java:38)
>>>>>>> at
>>>>>>> org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.runTests(RemoteTestRunner.java:467)
>>>>>>>  at
>>>>>>> org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.runTests(RemoteTestRunner.java:683)
>>>>>>> at
>>>>>>> org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.run(RemoteTestRunner.java:390)
>>>>>>>  at
>>>>>>> org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.main(RemoteTestRunner.java:197)
>>>>>>>
>>>>>>>
>>>>>>> Anyone has idea?
>>>>>>>
>>>>>>> BRs
>>>>>>> Geelong
>>>>>>>
>>>>>>> --
>>>>>>> From Good To Great
>>>>>>>
>>>>>>
>>>>>>
>>>>>
>>>>>
>>>>> --
>>>>> From Good To Great
>>>>>
>>>>
>>>>
>>>
>>>
>>> --
>>> From Good To Great
>>>
>>
>>
>
>
> --
> From Good To Great
>

Mime
View raw message