hadoop-hdfs-user mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From 姚吉龙 <geelong...@gmail.com>
Subject Re: Errors about MRunit
Date Sun, 21 Apr 2013 03:33:01 GMT
How to confirm this?



2013/4/21 Jagat Singh <jagatsingh@gmail.com>

> Can you please confirm if you are not mixing old and new mapreduce API
>
>
> On Sun, Apr 21, 2013 at 12:52 PM, 姚吉龙 <geelongyao@gmail.com> wrote:
>
>> Thanks
>> But I am confused about the MRunit. How to use MRunit without maven
>> I am just following the tutorial:
>> https://cwiki.apache.org/confluence/display/MRUNIT/MRUnit+Tutorial
>>
>> My test code  is below, the maper and reducer is in another project
>> linked in my testing project
>>
>>
>>
>> import java.io.IOException;
>> import java.util.ArrayList;
>> import java.util.List;
>> import junit.framework.TestCase;
>> import org.apache.hadoop.io.IntWritable;
>> import org.apache.hadoop.io.LongWritable;
>> import org.apache.hadoop.io.Text;
>> import org.apache.hadoop.mapred.lib.IdentityMapper;
>> import org.apache.hadoop.mapred.Mapper;
>> import org.apache.hadoop.mrunit.mapreduce.MapDriver;
>> import org.apache.hadoop.mrunit.mapreduce.MapReduceDriver;
>> import org.apache.hadoop.mrunit.mapreduce.ReduceDriver;
>> import org.apache.hadoop.mrunit.types.Pair;
>> import org.junit.Assert;
>> import org.junit.Before;
>> import org.junit.Test;
>>
>> public class UnitTest extends TestCase{
>>   MapDriver<Object, Text, Text, Text> mapDriver;
>>   ReduceDriver<Text, Text, Text, Text> reduceDriver;
>>   MapReduceDriver<Object, Text, Text, Text, Text, Text> mapReduceDriver;
>>    @Before
>>   public void setUp() {
>> carTest.FirstMapper mapper = new carTest.FirstMapper();
>>  carTest.FirstReducer reducer = new carTest.FirstReducer();
>>     mapDriver = MapDriver.newMapDriver(mapper);;
>>     reduceDriver = ReduceDriver.newReduceDriver(reducer);
>>     mapReduceDriver = MapReduceDriver.newMapReduceDriver(mapper, reducer);
>>   }
>>   @Test
>>   public void testMapper() throws IOException {
>>     mapDriver.withInput(new Object(), new Text(
>>
>>  "199397,32100000000000000800000120110131014195,321000000000000008,20110131170958,00,,苏K16423,2,,34.00,,,,460,,,K33,02,,1710048437180058,476096"));
>>     mapDriver.withOutput(new Text("苏K16423"), new
>> Text("321000000000000008"));
>>     mapDriver.runTest();
>>
>>  //"199397","32100000000000000800000120110131014195","321000000000000008","20110131170958","00","","苏K16423","2","","34.00","","","","460","","","K33","02","","1710048437180058","476096"
>>
>>   }
>>   @Test
>>   public void testReducer() throws IOException {
>>     List<Text> values = new ArrayList<Text>();
>>     values.add(new
>> Text("199397,32100000000000000800000120110131014195,321000000000000005,20110131172000,00,,苏K16423,2,,34.00,,,,460,,,K33,02,,1710048437180058,476096"));
>>   //  values.add(new Text("1"));
>>     reduceDriver.withInput(new Text("6"), values);
>>     reduceDriver.withOutput(new Text("6"), new Text("2"));
>>     reduceDriver.runTest();
>>   }
>>
>>
>>   }
>>
>> While, always failed to pass
>> [image: 内嵌图片 1]
>> Here is the trace
>>
>> java.lang.IncompatibleClassChangeError: Found class
>> org.apache.hadoop.mapreduce.TaskInputOutputContext, but interface was
>> expected
>> at
>> org.apache.hadoop.mrunit.mapreduce.mock.MockContextWrapper.createCommon(MockContextWrapper.java:53)
>>  at
>> org.apache.hadoop.mrunit.mapreduce.mock.MockMapContextWrapper.create(MockMapContextWrapper.java:70)
>> at
>> org.apache.hadoop.mrunit.mapreduce.mock.MockMapContextWrapper.<init>(MockMapContextWrapper.java:62)
>>  at org.apache.hadoop.mrunit.mapreduce.MapDriver.run(MapDriver.java:217)
>> at org.apache.hadoop.mrunit.MapDriverBase.runTest(MapDriverBase.java:150)
>>  at org.apache.hadoop.mrunit.TestDriver.runTest(TestDriver.java:137)
>> at UnitTest.testMapper(UnitTest.java:37)
>>  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>> at
>> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
>>  at
>> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
>> at java.lang.reflect.Method.invoke(Method.java:597)
>>  at junit.framework.TestCase.runTest(TestCase.java:168)
>> at junit.framework.TestCase.runBare(TestCase.java:134)
>>  at junit.framework.TestResult$1.protect(TestResult.java:110)
>> at junit.framework.TestResult.runProtected(TestResult.java:128)
>>  at junit.framework.TestResult.run(TestResult.java:113)
>> at junit.framework.TestCase.run(TestCase.java:124)
>>  at junit.framework.TestSuite.runTest(TestSuite.java:232)
>> at junit.framework.TestSuite.run(TestSuite.java:227)
>>  at
>> org.junit.internal.runners.JUnit38ClassRunner.run(JUnit38ClassRunner.java:79)
>> at
>> org.eclipse.jdt.internal.junit4.runner.JUnit4TestReference.run(JUnit4TestReference.java:50)
>>  at
>> org.eclipse.jdt.internal.junit.runner.TestExecution.run(TestExecution.java:38)
>> at
>> org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.runTests(RemoteTestRunner.java:467)
>>  at
>> org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.runTests(RemoteTestRunner.java:683)
>> at
>> org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.run(RemoteTestRunner.java:390)
>>  at
>> org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.main(RemoteTestRunner.java:197)
>>
>> Any body help to figure out these?
>> I am going crazy
>>
>>
>> BRs
>> Geelong
>>
>>
>>
>> 2013/4/21 Rishi Yadav <rishi@infoobjects.com>
>>
>>> Maven is not necessary to use hadoop or MRUnit. That being said I am a
>>> big fan of maven. Please find attached wordcount with maven.
>>>
>>>
>>>
>>> On Sat, Apr 20, 2013 at 7:17 PM, 姚吉龙 <geelongyao@gmail.com> wrote:
>>>
>>>> Thank for your reply
>>>> But I think the main problem is that I do konw how to use maven with
>>>> hadoop or MRunit
>>>> Any body can give me a example for MRunit project
>>>>
>>>>
>>>>
>>>> 2013/4/21 Rishi Yadav <rishi@infoobjects.com>
>>>>
>>>>> your problem is simple, you are mixing mapred (old api) and
>>>>> mapreduce(new api) libraries. MRUnit has implementation for both apis.
>>>>>
>>>>> Here's an example of WordCountTest with use of new api.
>>>>>
>>>>>
>>>>> ----------------------------------------------------------------------------------------------------------------------
>>>>>
>>>>> package com.infoobjects.hadoop.wc;
>>>>>
>>>>>
>>>>> import java.util.ArrayList;
>>>>>
>>>>> import java.util.List;
>>>>>
>>>>>
>>>>> import org.apache.hadoop.io.IntWritable;
>>>>>
>>>>> import org.apache.hadoop.io.LongWritable;
>>>>>
>>>>> import org.apache.hadoop.io.Text;
>>>>>
>>>>> import org.apache.hadoop.mrunit.mapreduce.MapDriver;
>>>>>
>>>>> import org.apache.hadoop.mrunit.mapreduce.MapReduceDriver;
>>>>>
>>>>> import org.apache.hadoop.mrunit.mapreduce.ReduceDriver;
>>>>>
>>>>> import org.junit.Before;
>>>>>
>>>>> import org.junit.Test;
>>>>>
>>>>>
>>>>> public class WordCountTest {
>>>>>
>>>>> MapDriver<LongWritable, Text, Text, IntWritable> mapDriver;
>>>>>
>>>>> ReduceDriver<Text, IntWritable, Text, IntWritable> reduceDriver;
>>>>>
>>>>> MapReduceDriver<LongWritable, Text, Text, IntWritable, Text,
>>>>> IntWritable> mapReduceDriver;
>>>>>
>>>>>
>>>>>  @Before
>>>>>
>>>>> public void init() {
>>>>>
>>>>>  WordMapper mapper = new WordMapper();
>>>>>
>>>>>  WordReducer reducer = new WordReducer();
>>>>>
>>>>>  mapDriver = new MapDriver<LongWritable, Text, Text, IntWritable>();
>>>>>
>>>>>  mapDriver.setMapper(mapper);
>>>>>
>>>>>     reduceDriver = ReduceDriver.newReduceDriver(reducer);
>>>>>
>>>>>     mapReduceDriver = MapReduceDriver.newMapReduceDriver(mapper,
>>>>> reducer);
>>>>>
>>>>>  }
>>>>>
>>>>>
>>>>>  @Test
>>>>>
>>>>> public void testMapper() {
>>>>>
>>>>>  mapDriver.withInput(new LongWritable(1), new Text("foo bar"));
>>>>>
>>>>>  mapDriver.withOutput(new Text("foo"), new IntWritable(1));
>>>>>
>>>>>  mapDriver.withOutput(new Text("bar"), new IntWritable(1));
>>>>>
>>>>>  mapDriver.runTest();
>>>>>
>>>>> }
>>>>>
>>>>>     @Test
>>>>>
>>>>>   public void testReducer() {
>>>>>
>>>>>     List<IntWritable> values = new ArrayList<IntWritable>();
>>>>>
>>>>>     values.add(new IntWritable(1));
>>>>>
>>>>>     values.add(new IntWritable(1));
>>>>>
>>>>>     reduceDriver.withInput(new Text("foo"), values);
>>>>>
>>>>>     reduceDriver.withOutput(new Text("foo"), new IntWritable(2));
>>>>>
>>>>>     reduceDriver.runTest();
>>>>>
>>>>>   }
>>>>>
>>>>>
>>>>>
>>>>>   @Test
>>>>>
>>>>>   public void testMapReduce() {
>>>>>
>>>>>    mapReduceDriver.withInput(new LongWritable(1), new Text("brian
>>>>> felix"));
>>>>>
>>>>>    mapReduceDriver.withOutput(new Text("foo"), new IntWritable(1));
>>>>>
>>>>>    mapReduceDriver.withOutput(new Text("bar"), new IntWritable(1));
>>>>>
>>>>>    mapReduceDriver.runTest();
>>>>>
>>>>>   }
>>>>>
>>>>>
>>>>> }
>>>>>
>>>>> Thanks and Regards,
>>>>>
>>>>> Rishi Yadav
>>>>>
>>>>> (o) 408.988.2000x113 ||  (f) 408.716.2726
>>>>>
>>>>> InfoObjects Inc || http://www.infoobjects.com *(Big Data Solutions)*
>>>>>
>>>>> *INC 500 Fastest growing company in 2012 || 2011*
>>>>>
>>>>> *Best Place to work in Bay Area 2012 - *SF Business Times and the
>>>>> Silicon Valley / San Jose Business Journal
>>>>>
>>>>> 2041 Mission College Boulevard, #280 || Santa Clara, CA 95054
>>>>>
>>>>>
>>>>>
>>>>>
>>>>> On Sat, Apr 20, 2013 at 7:14 AM, 姚吉龙 <geelongyao@gmail.com>
wrote:
>>>>>
>>>>>> This is what I got form my eclipse. Why still errors about the lib
>>>>>> from hadoop
>>>>>> [image: 内嵌图片 1][image: 内嵌图片 2]
>>>>>> anybody tell me how to use MRunit and Maven
>>>>>>
>>>>>>
>>>>>> 2013/4/20 Hemanth Yamijala <yhemanth@thoughtworks.com>
>>>>>>
>>>>>>> Hi,
>>>>>>>
>>>>>>> If your goal is to use the new API, I am able to get it to work
with
>>>>>>> the following maven configuration:
>>>>>>>
>>>>>>>     <dependency>
>>>>>>>       <groupId>org.apache.mrunit</groupId>
>>>>>>>       <artifactId>mrunit</artifactId>
>>>>>>>       <version>0.9.0-incubating</version>
>>>>>>>       <classifier>hadoop1</classifier>
>>>>>>>     </dependency>
>>>>>>>
>>>>>>> If I switch with classifier hadoop2, I get the same errors as
what
>>>>>>> you facing.
>>>>>>>
>>>>>>> Thanks
>>>>>>> Hemanth
>>>>>>>
>>>>>>>
>>>>>>> On Sat, Apr 20, 2013 at 3:42 PM, 姚吉龙 <geelongyao@gmail.com>
wrote:
>>>>>>>
>>>>>>>> Hi Everyone
>>>>>>>>
>>>>>>>> I am testing my MR programe with MRunit, it's version
>>>>>>>> is mrunit-0.9.0-incubating-hadoop2. My hadoop version is
1.0.4
>>>>>>>> The error trace is below:
>>>>>>>>
>>>>>>>> java.lang.IncompatibleClassChangeError: Found class
>>>>>>>> org.apache.hadoop.mapreduce.TaskInputOutputContext, but interface
was
>>>>>>>> expected
>>>>>>>> at
>>>>>>>> org.apache.hadoop.mrunit.mapreduce.mock.MockContextWrapper.createCommon(MockContextWrapper.java:53)
>>>>>>>>  at
>>>>>>>> org.apache.hadoop.mrunit.mapreduce.mock.MockMapContextWrapper.create(MockMapContextWrapper.java:70)
>>>>>>>> at
>>>>>>>> org.apache.hadoop.mrunit.mapreduce.mock.MockMapContextWrapper.<init>(MockMapContextWrapper.java:62)
>>>>>>>>  at
>>>>>>>> org.apache.hadoop.mrunit.mapreduce.MapDriver.run(MapDriver.java:217)
>>>>>>>> at
>>>>>>>> org.apache.hadoop.mrunit.MapDriverBase.runTest(MapDriverBase.java:150)
>>>>>>>>  at
>>>>>>>> org.apache.hadoop.mrunit.TestDriver.runTest(TestDriver.java:137)
>>>>>>>> at UnitTest.testMapper(UnitTest.java:41)
>>>>>>>>  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>>>>>>>> at
>>>>>>>> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
>>>>>>>>  at
>>>>>>>> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
>>>>>>>> at java.lang.reflect.Method.invoke(Method.java:597)
>>>>>>>>  at junit.framework.TestCase.runTest(TestCase.java:168)
>>>>>>>> at junit.framework.TestCase.runBare(TestCase.java:134)
>>>>>>>>  at junit.framework.TestResult$1.protect(TestResult.java:110)
>>>>>>>> at junit.framework.TestResult.runProtected(TestResult.java:128)
>>>>>>>>  at junit.framework.TestResult.run(TestResult.java:113)
>>>>>>>> at junit.framework.TestCase.run(TestCase.java:124)
>>>>>>>>  at junit.framework.TestSuite.runTest(TestSuite.java:232)
>>>>>>>> at junit.framework.TestSuite.run(TestSuite.java:227)
>>>>>>>>  at
>>>>>>>> org.junit.internal.runners.JUnit38ClassRunner.run(JUnit38ClassRunner.java:79)
>>>>>>>> at
>>>>>>>> org.eclipse.jdt.internal.junit4.runner.JUnit4TestReference.run(JUnit4TestReference.java:50)
>>>>>>>>  at
>>>>>>>> org.eclipse.jdt.internal.junit.runner.TestExecution.run(TestExecution.java:38)
>>>>>>>> at
>>>>>>>> org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.runTests(RemoteTestRunner.java:467)
>>>>>>>>  at
>>>>>>>> org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.runTests(RemoteTestRunner.java:683)
>>>>>>>> at
>>>>>>>> org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.run(RemoteTestRunner.java:390)
>>>>>>>>  at
>>>>>>>> org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.main(RemoteTestRunner.java:197)
>>>>>>>>
>>>>>>>>
>>>>>>>> Anyone has idea?
>>>>>>>>
>>>>>>>> BRs
>>>>>>>> Geelong
>>>>>>>>
>>>>>>>> --
>>>>>>>> From Good To Great
>>>>>>>>
>>>>>>>
>>>>>>>
>>>>>>
>>>>>>
>>>>>> --
>>>>>> From Good To Great
>>>>>>
>>>>>
>>>>>
>>>>
>>>>
>>>> --
>>>> From Good To Great
>>>>
>>>
>>>
>>
>>
>> --
>> From Good To Great
>>
>
>


-- 
>From Good To Great

Mime
View raw message