hadoop-mapreduce-user mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From 姚吉龙 <geelong...@gmail.com>
Subject Re: Errors about MRunit
Date Sun, 21 Apr 2013 05:16:33 GMT
I am the newer for MRunit and Maven
I just google this issue and find a solution:
http://stackoverflow.com/questions/12444663/hadoop-mrunit-throws-exception
it said that we need to change the pon.xml, I am wondering whether this
file is in mrunit-0.9.0-incubating-hadoop2.jar
[image: 内嵌图片 1][image: 内嵌图片 2]

but it still not working


BRs
Geelong





2013/4/21 姚吉龙 <geelongyao@gmail.com>

> How to confirm this?
>
>
>
> 2013/4/21 Jagat Singh <jagatsingh@gmail.com>
>
>> Can you please confirm if you are not mixing old and new mapreduce API
>>
>>
>> On Sun, Apr 21, 2013 at 12:52 PM, 姚吉龙 <geelongyao@gmail.com> wrote:
>>
>>> Thanks
>>> But I am confused about the MRunit. How to use MRunit without maven
>>> I am just following the tutorial:
>>> https://cwiki.apache.org/confluence/display/MRUNIT/MRUnit+Tutorial
>>>
>>> My test code  is below, the maper and reducer is in another project
>>> linked in my testing project
>>>
>>>
>>>
>>> import java.io.IOException;
>>> import java.util.ArrayList;
>>> import java.util.List;
>>> import junit.framework.TestCase;
>>> import org.apache.hadoop.io.IntWritable;
>>> import org.apache.hadoop.io.LongWritable;
>>> import org.apache.hadoop.io.Text;
>>> import org.apache.hadoop.mapred.lib.IdentityMapper;
>>> import org.apache.hadoop.mapred.Mapper;
>>> import org.apache.hadoop.mrunit.mapreduce.MapDriver;
>>> import org.apache.hadoop.mrunit.mapreduce.MapReduceDriver;
>>> import org.apache.hadoop.mrunit.mapreduce.ReduceDriver;
>>> import org.apache.hadoop.mrunit.types.Pair;
>>> import org.junit.Assert;
>>> import org.junit.Before;
>>> import org.junit.Test;
>>>
>>> public class UnitTest extends TestCase{
>>>   MapDriver<Object, Text, Text, Text> mapDriver;
>>>   ReduceDriver<Text, Text, Text, Text> reduceDriver;
>>>   MapReduceDriver<Object, Text, Text, Text, Text, Text> mapReduceDriver;
>>>    @Before
>>>   public void setUp() {
>>> carTest.FirstMapper mapper = new carTest.FirstMapper();
>>>  carTest.FirstReducer reducer = new carTest.FirstReducer();
>>>     mapDriver = MapDriver.newMapDriver(mapper);;
>>>     reduceDriver = ReduceDriver.newReduceDriver(reducer);
>>>     mapReduceDriver = MapReduceDriver.newMapReduceDriver(mapper,
>>> reducer);
>>>   }
>>>   @Test
>>>   public void testMapper() throws IOException {
>>>     mapDriver.withInput(new Object(), new Text(
>>>
>>>  "199397,32100000000000000800000120110131014195,321000000000000008,20110131170958,00,,苏K16423,2,,34.00,,,,460,,,K33,02,,1710048437180058,476096"));
>>>     mapDriver.withOutput(new Text("苏K16423"), new
>>> Text("321000000000000008"));
>>>     mapDriver.runTest();
>>>
>>>  //"199397","32100000000000000800000120110131014195","321000000000000008","20110131170958","00","","苏K16423","2","","34.00","","","","460","","","K33","02","","1710048437180058","476096"
>>>
>>>   }
>>>   @Test
>>>   public void testReducer() throws IOException {
>>>     List<Text> values = new ArrayList<Text>();
>>>     values.add(new
>>> Text("199397,32100000000000000800000120110131014195,321000000000000005,20110131172000,00,,苏K16423,2,,34.00,,,,460,,,K33,02,,1710048437180058,476096"));
>>>   //  values.add(new Text("1"));
>>>     reduceDriver.withInput(new Text("6"), values);
>>>     reduceDriver.withOutput(new Text("6"), new Text("2"));
>>>     reduceDriver.runTest();
>>>   }
>>>
>>>
>>>   }
>>>
>>> While, always failed to pass
>>> [image: 内嵌图片 1]
>>> Here is the trace
>>>
>>> java.lang.IncompatibleClassChangeError: Found class
>>> org.apache.hadoop.mapreduce.TaskInputOutputContext, but interface was
>>> expected
>>> at
>>> org.apache.hadoop.mrunit.mapreduce.mock.MockContextWrapper.createCommon(MockContextWrapper.java:53)
>>>  at
>>> org.apache.hadoop.mrunit.mapreduce.mock.MockMapContextWrapper.create(MockMapContextWrapper.java:70)
>>> at
>>> org.apache.hadoop.mrunit.mapreduce.mock.MockMapContextWrapper.<init>(MockMapContextWrapper.java:62)
>>>  at org.apache.hadoop.mrunit.mapreduce.MapDriver.run(MapDriver.java:217)
>>> at org.apache.hadoop.mrunit.MapDriverBase.runTest(MapDriverBase.java:150)
>>>  at org.apache.hadoop.mrunit.TestDriver.runTest(TestDriver.java:137)
>>> at UnitTest.testMapper(UnitTest.java:37)
>>>  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>>> at
>>> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
>>>  at
>>> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
>>> at java.lang.reflect.Method.invoke(Method.java:597)
>>>  at junit.framework.TestCase.runTest(TestCase.java:168)
>>> at junit.framework.TestCase.runBare(TestCase.java:134)
>>>  at junit.framework.TestResult$1.protect(TestResult.java:110)
>>> at junit.framework.TestResult.runProtected(TestResult.java:128)
>>>  at junit.framework.TestResult.run(TestResult.java:113)
>>> at junit.framework.TestCase.run(TestCase.java:124)
>>>  at junit.framework.TestSuite.runTest(TestSuite.java:232)
>>> at junit.framework.TestSuite.run(TestSuite.java:227)
>>>  at
>>> org.junit.internal.runners.JUnit38ClassRunner.run(JUnit38ClassRunner.java:79)
>>> at
>>> org.eclipse.jdt.internal.junit4.runner.JUnit4TestReference.run(JUnit4TestReference.java:50)
>>>  at
>>> org.eclipse.jdt.internal.junit.runner.TestExecution.run(TestExecution.java:38)
>>> at
>>> org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.runTests(RemoteTestRunner.java:467)
>>>  at
>>> org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.runTests(RemoteTestRunner.java:683)
>>> at
>>> org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.run(RemoteTestRunner.java:390)
>>>  at
>>> org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.main(RemoteTestRunner.java:197)
>>>
>>> Any body help to figure out these?
>>> I am going crazy
>>>
>>>
>>> BRs
>>> Geelong
>>>
>>>
>>>
>>> 2013/4/21 Rishi Yadav <rishi@infoobjects.com>
>>>
>>>> Maven is not necessary to use hadoop or MRUnit. That being said I am a
>>>> big fan of maven. Please find attached wordcount with maven.
>>>>
>>>>
>>>>
>>>> On Sat, Apr 20, 2013 at 7:17 PM, 姚吉龙 <geelongyao@gmail.com> wrote:
>>>>
>>>>> Thank for your reply
>>>>> But I think the main problem is that I do konw how to use maven with
>>>>> hadoop or MRunit
>>>>> Any body can give me a example for MRunit project
>>>>>
>>>>>
>>>>>
>>>>> 2013/4/21 Rishi Yadav <rishi@infoobjects.com>
>>>>>
>>>>>> your problem is simple, you are mixing mapred (old api) and
>>>>>> mapreduce(new api) libraries. MRUnit has implementation for both
apis.
>>>>>>
>>>>>> Here's an example of WordCountTest with use of new api.
>>>>>>
>>>>>>
>>>>>> ----------------------------------------------------------------------------------------------------------------------
>>>>>>
>>>>>> package com.infoobjects.hadoop.wc;
>>>>>>
>>>>>>
>>>>>> import java.util.ArrayList;
>>>>>>
>>>>>> import java.util.List;
>>>>>>
>>>>>>
>>>>>> import org.apache.hadoop.io.IntWritable;
>>>>>>
>>>>>> import org.apache.hadoop.io.LongWritable;
>>>>>>
>>>>>> import org.apache.hadoop.io.Text;
>>>>>>
>>>>>> import org.apache.hadoop.mrunit.mapreduce.MapDriver;
>>>>>>
>>>>>> import org.apache.hadoop.mrunit.mapreduce.MapReduceDriver;
>>>>>>
>>>>>> import org.apache.hadoop.mrunit.mapreduce.ReduceDriver;
>>>>>>
>>>>>> import org.junit.Before;
>>>>>>
>>>>>> import org.junit.Test;
>>>>>>
>>>>>>
>>>>>> public class WordCountTest {
>>>>>>
>>>>>> MapDriver<LongWritable, Text, Text, IntWritable> mapDriver;
>>>>>>
>>>>>> ReduceDriver<Text, IntWritable, Text, IntWritable> reduceDriver;
>>>>>>
>>>>>> MapReduceDriver<LongWritable, Text, Text, IntWritable, Text,
>>>>>> IntWritable> mapReduceDriver;
>>>>>>
>>>>>>
>>>>>>  @Before
>>>>>>
>>>>>> public void init() {
>>>>>>
>>>>>>  WordMapper mapper = new WordMapper();
>>>>>>
>>>>>>  WordReducer reducer = new WordReducer();
>>>>>>
>>>>>>  mapDriver = new MapDriver<LongWritable, Text, Text, IntWritable>();
>>>>>>
>>>>>>  mapDriver.setMapper(mapper);
>>>>>>
>>>>>>     reduceDriver = ReduceDriver.newReduceDriver(reducer);
>>>>>>
>>>>>>     mapReduceDriver = MapReduceDriver.newMapReduceDriver(mapper,
>>>>>> reducer);
>>>>>>
>>>>>>  }
>>>>>>
>>>>>>
>>>>>>  @Test
>>>>>>
>>>>>> public void testMapper() {
>>>>>>
>>>>>>  mapDriver.withInput(new LongWritable(1), new Text("foo bar"));
>>>>>>
>>>>>>  mapDriver.withOutput(new Text("foo"), new IntWritable(1));
>>>>>>
>>>>>>  mapDriver.withOutput(new Text("bar"), new IntWritable(1));
>>>>>>
>>>>>>  mapDriver.runTest();
>>>>>>
>>>>>> }
>>>>>>
>>>>>>     @Test
>>>>>>
>>>>>>   public void testReducer() {
>>>>>>
>>>>>>     List<IntWritable> values = new ArrayList<IntWritable>();
>>>>>>
>>>>>>     values.add(new IntWritable(1));
>>>>>>
>>>>>>     values.add(new IntWritable(1));
>>>>>>
>>>>>>     reduceDriver.withInput(new Text("foo"), values);
>>>>>>
>>>>>>     reduceDriver.withOutput(new Text("foo"), new IntWritable(2));
>>>>>>
>>>>>>     reduceDriver.runTest();
>>>>>>
>>>>>>   }
>>>>>>
>>>>>>
>>>>>>
>>>>>>   @Test
>>>>>>
>>>>>>   public void testMapReduce() {
>>>>>>
>>>>>>    mapReduceDriver.withInput(new LongWritable(1), new Text("brian
>>>>>> felix"));
>>>>>>
>>>>>>    mapReduceDriver.withOutput(new Text("foo"), new IntWritable(1));
>>>>>>
>>>>>>    mapReduceDriver.withOutput(new Text("bar"), new IntWritable(1));
>>>>>>
>>>>>>    mapReduceDriver.runTest();
>>>>>>
>>>>>>   }
>>>>>>
>>>>>>
>>>>>> }
>>>>>>
>>>>>> Thanks and Regards,
>>>>>>
>>>>>> Rishi Yadav
>>>>>>
>>>>>> (o) 408.988.2000x113 ||  (f) 408.716.2726
>>>>>>
>>>>>> InfoObjects Inc || http://www.infoobjects.com *(Big Data Solutions)*
>>>>>>
>>>>>> *INC 500 Fastest growing company in 2012 || 2011*
>>>>>>
>>>>>> *Best Place to work in Bay Area 2012 - *SF Business Times and the
>>>>>> Silicon Valley / San Jose Business Journal
>>>>>>
>>>>>> 2041 Mission College Boulevard, #280 || Santa Clara, CA 95054
>>>>>>
>>>>>>
>>>>>>
>>>>>>
>>>>>> On Sat, Apr 20, 2013 at 7:14 AM, 姚吉龙 <geelongyao@gmail.com>
wrote:
>>>>>>
>>>>>>> This is what I got form my eclipse. Why still errors about the
lib
>>>>>>> from hadoop
>>>>>>> [image: 内嵌图片 1][image: 内嵌图片 2]
>>>>>>> anybody tell me how to use MRunit and Maven
>>>>>>>
>>>>>>>
>>>>>>> 2013/4/20 Hemanth Yamijala <yhemanth@thoughtworks.com>
>>>>>>>
>>>>>>>> Hi,
>>>>>>>>
>>>>>>>> If your goal is to use the new API, I am able to get it to
work
>>>>>>>> with the following maven configuration:
>>>>>>>>
>>>>>>>>     <dependency>
>>>>>>>>       <groupId>org.apache.mrunit</groupId>
>>>>>>>>       <artifactId>mrunit</artifactId>
>>>>>>>>       <version>0.9.0-incubating</version>
>>>>>>>>       <classifier>hadoop1</classifier>
>>>>>>>>     </dependency>
>>>>>>>>
>>>>>>>> If I switch with classifier hadoop2, I get the same errors
as what
>>>>>>>> you facing.
>>>>>>>>
>>>>>>>> Thanks
>>>>>>>> Hemanth
>>>>>>>>
>>>>>>>>
>>>>>>>> On Sat, Apr 20, 2013 at 3:42 PM, 姚吉龙 <geelongyao@gmail.com>
wrote:
>>>>>>>>
>>>>>>>>> Hi Everyone
>>>>>>>>>
>>>>>>>>> I am testing my MR programe with MRunit, it's version
>>>>>>>>> is mrunit-0.9.0-incubating-hadoop2. My hadoop version
is 1.0.4
>>>>>>>>> The error trace is below:
>>>>>>>>>
>>>>>>>>> java.lang.IncompatibleClassChangeError: Found class
>>>>>>>>> org.apache.hadoop.mapreduce.TaskInputOutputContext, but
interface was
>>>>>>>>> expected
>>>>>>>>> at
>>>>>>>>> org.apache.hadoop.mrunit.mapreduce.mock.MockContextWrapper.createCommon(MockContextWrapper.java:53)
>>>>>>>>>  at
>>>>>>>>> org.apache.hadoop.mrunit.mapreduce.mock.MockMapContextWrapper.create(MockMapContextWrapper.java:70)
>>>>>>>>> at
>>>>>>>>> org.apache.hadoop.mrunit.mapreduce.mock.MockMapContextWrapper.<init>(MockMapContextWrapper.java:62)
>>>>>>>>>  at
>>>>>>>>> org.apache.hadoop.mrunit.mapreduce.MapDriver.run(MapDriver.java:217)
>>>>>>>>> at
>>>>>>>>> org.apache.hadoop.mrunit.MapDriverBase.runTest(MapDriverBase.java:150)
>>>>>>>>>  at
>>>>>>>>> org.apache.hadoop.mrunit.TestDriver.runTest(TestDriver.java:137)
>>>>>>>>> at UnitTest.testMapper(UnitTest.java:41)
>>>>>>>>>  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
Method)
>>>>>>>>> at
>>>>>>>>> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
>>>>>>>>>  at
>>>>>>>>> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
>>>>>>>>> at java.lang.reflect.Method.invoke(Method.java:597)
>>>>>>>>>  at junit.framework.TestCase.runTest(TestCase.java:168)
>>>>>>>>> at junit.framework.TestCase.runBare(TestCase.java:134)
>>>>>>>>>  at junit.framework.TestResult$1.protect(TestResult.java:110)
>>>>>>>>> at junit.framework.TestResult.runProtected(TestResult.java:128)
>>>>>>>>>  at junit.framework.TestResult.run(TestResult.java:113)
>>>>>>>>> at junit.framework.TestCase.run(TestCase.java:124)
>>>>>>>>>  at junit.framework.TestSuite.runTest(TestSuite.java:232)
>>>>>>>>> at junit.framework.TestSuite.run(TestSuite.java:227)
>>>>>>>>>  at
>>>>>>>>> org.junit.internal.runners.JUnit38ClassRunner.run(JUnit38ClassRunner.java:79)
>>>>>>>>> at
>>>>>>>>> org.eclipse.jdt.internal.junit4.runner.JUnit4TestReference.run(JUnit4TestReference.java:50)
>>>>>>>>>  at
>>>>>>>>> org.eclipse.jdt.internal.junit.runner.TestExecution.run(TestExecution.java:38)
>>>>>>>>> at
>>>>>>>>> org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.runTests(RemoteTestRunner.java:467)
>>>>>>>>>  at
>>>>>>>>> org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.runTests(RemoteTestRunner.java:683)
>>>>>>>>> at
>>>>>>>>> org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.run(RemoteTestRunner.java:390)
>>>>>>>>>  at
>>>>>>>>> org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.main(RemoteTestRunner.java:197)
>>>>>>>>>
>>>>>>>>>
>>>>>>>>> Anyone has idea?
>>>>>>>>>
>>>>>>>>> BRs
>>>>>>>>> Geelong
>>>>>>>>>
>>>>>>>>> --
>>>>>>>>> From Good To Great
>>>>>>>>>
>>>>>>>>
>>>>>>>>
>>>>>>>
>>>>>>>
>>>>>>> --
>>>>>>> From Good To Great
>>>>>>>
>>>>>>
>>>>>>
>>>>>
>>>>>
>>>>> --
>>>>> From Good To Great
>>>>>
>>>>
>>>>
>>>
>>>
>>> --
>>> From Good To Great
>>>
>>
>>
>
>
> --
> From Good To Great
>



-- 
>From Good To Great

Mime
View raw message