hadoop-user mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From Björn-Elmar Macek <ma...@cs.uni-kassel.de>
Subject Re: mortbay, huge files and the ulimit
Date Wed, 05 Sep 2012 12:02:49 GMT
Excuse me: in my last code section was some old code included. Here is 
it again stripped of deprecated code:

package uni.kassel.macek.rtprep;



import gnu.trove.iterator.TIntIterator;
import gnu.trove.map.hash.TIntObjectHashMap;
import gnu.trove.set.TIntSet;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Iterator;
import java.util.List;
import java.util.StringTokenizer;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.MapReduceBase;
import org.apache.hadoop.mapred.OutputCollector;
import org.apache.hadoop.mapred.Reporter;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.util.GenericOptionsParser;

public class RetweetApplication {

     public static class RetweetMapper1 extends Mapper<Object, Text, 
Text, Text> {

         public void map(Object key, Text value,
             Context context)
             throws IOException {

             String line = value.toString();

             Tweet aTweet = new Tweet(line);
             if(aTweet.getMention()!=null && 
aTweet.getMention().length()>0){
                 /*aTweet.setRT(0);
                 output.collect(new Text(aTweet.toString()), new 
Text(aTweet.toString()));*/
                 aTweet.setRT(1);
                 try {
                     context.write(new Text(aTweet.getMention()), new 
Text(aTweet.toString()));
                 } catch (InterruptedException e) {
                     // TODO Auto-generated catch block
                     e.printStackTrace();
                 }
             }
             aTweet.setRT(0);
             try {
                 context.write(new Text(aTweet.getAuthor()), new 
Text(aTweet.toString()));
             } catch (InterruptedException e) {
                 // TODO Auto-generated catch block
                 e.printStackTrace();
             }

         }
     }

public static class RetweetReducer1 extends Reducer<Text, Text, Text, 
Text> {

     public void reduce(Text atweet, Iterator<Text> tweets,
             Context context)
             throws IOException {
         List<Tweet> originals = new ArrayList<Tweet>();
         List<Tweet> retweets = new ArrayList<Tweet>();
         while(tweets.hasNext()){
             Tweet aTweet = new Tweet(tweets.toString());
             originals.add(aTweet);
             if(aTweet.getRT()>0) retweets.add(aTweet);
         }

         for(Tweet originalTweet : originals){
             for(Tweet aRetweet : retweets){
                 if(aRetweet.isRetweeted(originalTweet))
                 {
aRetweet.setReactionTime(originalTweet.getDate());
                     try {
                         context.write(
                                 atweet,
                                 new Text(aRetweet.toString()));
                     } catch (InterruptedException e) {
                         // TODO Auto-generated catch block
                         e.printStackTrace();
                     }
                 }
             }
         }
     }
}

   public static void main(String[] args) throws Exception {
     Configuration conf = new Configuration();
     String[] otherArgs = new GenericOptionsParser(conf, 
args).getRemainingArgs();
     if (otherArgs.length != 2) {
       System.err.println("Usage: wordcount <in> <out>");
       System.exit(2);
     }
     Job job = new Job(conf, "word count");
     job.setJarByClass(RetweetApplication.class);
     job.setMapperClass(RetweetMapper1.class);
     job.setReducerClass(RetweetReducer1.class);
     job.setOutputKeyClass(Text.class);
     job.setOutputValueClass(Text.class);
     FileInputFormat.addInputPath(job, new Path(otherArgs[0]));
     FileOutputFormat.setOutputPath(job, new Path(otherArgs[1]));
     System.exit(job.waitForCompletion(true) ? 0 : 1);
   }
}
Am 05.09.2012 13:56, schrieb Björn-Elmar Macek:
> Hello again,
>
> i just wanted to keep you updated, in case anybody reads this and is 
> interested in the mortbay-issue:
> I applied the wordcount example to a big input file and everything 
> worked fine. I watched the TT logs all the time and no single 
> mortbay-eof exception occurred. The only possible explanation for me 
> seemed to be, that my code somehow produces the exceptions. So i 
> started to keep my code as close to the wordcount example as possible:
>
> * first getting rid of any members within the reducer class, that may 
> take lots of memory.
> * using the Context class instead of OutputCollector (in Mapper and 
> Reducer)
> * removing any Comparator (and restructuring the code so it just 
> calculates the same as with the sorting)
> * using standart HashPartitioner
>
> Especially the first 2 points made my job succeed in the end, ALTHOUGH 
> the problem of "n-1 mortbay-exception-throwing TTs" still remains: one 
> node seems to complete all the work while the others are busy with 
> their exceptions (at least from what i can see in the log).
>
> I REALLY dont know what i can do, to make this job run through like 
> the wordcount example. I dont see any big difference between my 
> program and the example anymore.
>
> I added the code here:
>
> *Please* help me here.
>
> Best regards,
> Elmar
>
>
> I added the code here:
> ########################################
> package uni.kassel.macek.rtprep;
>
> import java.util.Date;
>
> import org.apache.hadoop.conf.Configuration;
> import org.apache.hadoop.conf.Configured;
> import org.apache.hadoop.fs.Path;
> import org.apache.hadoop.io.Text;
> import org.apache.hadoop.mapred.FileInputFormat;
> import org.apache.hadoop.mapred.FileOutputFormat;
> import org.apache.hadoop.mapred.JobClient;
> import org.apache.hadoop.mapred.JobConf;
> import org.apache.hadoop.mapred.TextOutputFormat;
> import org.apache.hadoop.util.Tool;
> import org.apache.hadoop.util.ToolRunner;
>
> public class RetweetApplication extends Configured implements Tool{
>
>     public static int DEBUG = -1;
>     private Configuration jobConf;
>
>     //static String INPUT = "/home/fb16/bmacek/HADOOP-INPUT";
>     //static String OUTPUT = "/home/fb16/bmacek/HADOOP-OUTPUT "+ (new 
> Date()).toString();
>
>     public static void main(String[] args) {
>          int res=-1;
>         try {
>             res = ToolRunner.run(new Configuration(), new 
> RetweetApplication(), args);
>         } catch (Exception e) {
>             // TODO Auto-generated catch block
>             e.printStackTrace();
>         }
>         System.exit(res);
>
>     }
>
>     public void setConf(Configuration conf) {
>         jobConf = conf;
>
>     }
>
>     public Configuration getConf() {
>         return jobConf;
>     }
>
>     public int run(String[] args) throws Exception {
>         JobClient client = new JobClient();
>         JobConf conf = new JobConf(RetweetApplication.class);
>
>
>         if(DEBUG > 0){
>             conf.set("mapred.job.tracker", "local");
>             conf.set("fs.default.name", "file:///");
>             conf.set("dfs.replication", "1");
>         }
>
>         //conf.set("mapred.child.java.opts","-Xmx3000m 
> -XX:+UseCompressedOops -XX:-UseGCOverheadLimit");
>         conf.set("mapred.child.ulimit","100000000");
>
>         FileInputFormat.setInputPaths(conf, new Path(args[0]));
>         FileOutputFormat.setOutputPath(conf, new Path(args[1]));
>
>
>         //conf.setOutputKeyClass(Text.class);
>         //conf.setOutputValueClass(Text.class);
>         conf.setMapOutputKeyClass(Text.class);
>         conf.setMapOutputValueClass(Text.class);
>
>         conf.setMapperClass(RetweetMapper.class);
>         //conf.setPartitionerClass(TweetPartitioner.class);
> //conf.setOutputKeyComparatorClass(TwitterValueGroupingComparator.class);
> //conf.setOutputValueGroupingComparator(TwitterKeyGroupingComparator.class); 
>
>         conf.setReducerClass(RetweetReducer.class);
>
>         conf.setOutputFormat(TextOutputFormat.class);
>
>         client.setConf(conf);
>         try {
>             JobClient.runJob(conf);
>         } catch (Exception e) {
>             e.printStackTrace();
>         }
>         return 0;
>     }
> }
> */
>
> package uni.kassel.macek.rtprep;
>
>
>
> import gnu.trove.iterator.TIntIterator;
> import gnu.trove.map.hash.TIntObjectHashMap;
> import gnu.trove.set.TIntSet;
>
> import java.io.IOException;
> import java.util.ArrayList;
> import java.util.Calendar;
> import java.util.Iterator;
> import java.util.List;
> import java.util.StringTokenizer;
>
> import org.apache.hadoop.conf.Configuration;
> import org.apache.hadoop.fs.Path;
> import org.apache.hadoop.io.IntWritable;
> import org.apache.hadoop.io.LongWritable;
> import org.apache.hadoop.io.Text;
> import org.apache.hadoop.mapred.MapReduceBase;
> import org.apache.hadoop.mapred.OutputCollector;
> import org.apache.hadoop.mapred.Reporter;
> import org.apache.hadoop.mapreduce.Job;
> import org.apache.hadoop.mapreduce.Mapper;
> import org.apache.hadoop.mapreduce.Reducer;
> import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
> import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
> import org.apache.hadoop.util.GenericOptionsParser;
>
> public class RetweetApplication {
>
>     public static class RetweetMapper1 extends Mapper<Object, Text, 
> Text, Text> {
>
>         public void map(Object key, Text value,
>             Context context)
>             throws IOException {
>
>             String line = value.toString();
>
>             Tweet aTweet = new Tweet(line);
>             if(aTweet.getMention()!=null && 
> aTweet.getMention().length()>0){
>                 /*aTweet.setRT(0);
>                 output.collect(new Text(aTweet.toString()), new 
> Text(aTweet.toString()));*/
>                 aTweet.setRT(1);
>                 try {
>                     context.write(new Text(aTweet.getMention()), new 
> Text(aTweet.toString()));
>                 } catch (InterruptedException e) {
>                     // TODO Auto-generated catch block
>                     e.printStackTrace();
>                 }
>             }
>             aTweet.setRT(0);
>             try {
>                 context.write(new Text(aTweet.getAuthor()), new 
> Text(aTweet.toString()));
>             } catch (InterruptedException e) {
>                 // TODO Auto-generated catch block
>                 e.printStackTrace();
>             }
>
>         }
>     }
>
> public static class RetweetReducer1 extends Reducer<Text, Text, Text, 
> Text> {
>
>     public void reduce(Text atweet, Iterator<Text> tweets,
>             Context context)
>             throws IOException {
>         List<Tweet> originals = new ArrayList<Tweet>();
>         List<Tweet> retweets = new ArrayList<Tweet>();
>         while(tweets.hasNext()){
>             Tweet aTweet = new Tweet(tweets.toString());
>             originals.add(aTweet);
>             if(aTweet.getRT()>0) retweets.add(aTweet);
>         }
>
>         for(Tweet originalTweet : originals){
>             for(Tweet aRetweet : retweets){
>                 if(aRetweet.isRetweeted(originalTweet))
>                 {
> aRetweet.setReactionTime(originalTweet.getDate());
>                     try {
>                         context.write(
>                                 atweet,
>                                 new Text(aRetweet.toString()));
>                     } catch (InterruptedException e) {
>                         // TODO Auto-generated catch block
>                         e.printStackTrace();
>                     }
>                 }
>             }
>         }
>     }
> }
>
>   public static void main(String[] args) throws Exception {
>     Configuration conf = new Configuration();
>     String[] otherArgs = new GenericOptionsParser(conf, 
> args).getRemainingArgs();
>     if (otherArgs.length != 2) {
>       System.err.println("Usage: wordcount <in> <out>");
>       System.exit(2);
>     }
>     Job job = new Job(conf, "tweets");
>     job.setJarByClass(RetweetApplication.class);
>     job.setMapperClass(RetweetMapper1.class);
>     job.setReducerClass(RetweetReducer1.class);
>     job.setOutputKeyClass(Text.class);
>     job.setOutputValueClass(Text.class);
>     FileInputFormat.addInputPath(job, new Path(otherArgs[0]));
>     FileOutputFormat.setOutputPath(job, new Path(otherArgs[1]));
>     System.exit(job.waitForCompletion(true) ? 0 : 1);
>   }
> }
> Am 31.08.2012 14:08, schrieb Björn-Elmar Macek:
>> Hi all,
>>
>> i got some more information:
>>
>> It is always only one TT, where mortbay errors do not occur: all 
>> other slaves fail relatively fast after the reduce process started: 
>> maybe it is a comparator issue (since the operation is MAPRED_SHUFFLE)
>> And it has nothing to do with the local configuration of the server, 
>> since
>> 1) if i rerun the job on the same set of servers, the ONE working 
>> differs from one job to another.
>> 2) the same happens even when i am using a completely different set 
>> of servers.
>>
>> Who is capable to solve this unsolved mystery? :D
>>
>> Am 30.08.2012 12:27, schrieb Björn-Elmar Macek:
>>> Hi,
>>>
>>> is there anybody, who knows more about this issue: it has already 
>>> been recently marked here:
>>> https://issues.apache.org/jira/browse/MAPREDUCE-5
>>>
>>> I really want to do something about it, if i knew how... i tried so 
>>> many different setup parameters and JVM options and nothing did the 
>>> trick...
>>>
>>> It was opened 2009 and gets still updated (2012): I mean - this does 
>>> NOT look OK, does it? The whole TT seems like a big time waster:
>>>
>>> 2012-08-30 12:05:22,582 INFO 
>>> org.apache.hadoop.mapred.TaskTracker.clienttrace: src: 
>>> 141.51.205.130:35497, dest: 141.51.205.126:39586, bytes: 146086410, 
>>> op: MAPRED_SHUFFLE, cliID: attempt_201208301039_0001_m_000189_0, 
>>> duration: 5456122852
>>> 2012-08-30 12:05:56,275 INFO org.mortbay.log: 
>>> org.mortbay.io.nio.SelectorManager$SelectSet@6bfd67ba JVM BUG(s) - 
>>> injecting delay2 times
>>> 2012-08-30 12:05:56,276 INFO org.mortbay.log: 
>>> org.mortbay.io.nio.SelectorManager$SelectSet@6bfd67ba JVM BUG(s) - 
>>> recreating selector 2 times, canceled keys 70 times
>>> 2012-08-30 12:08:39,236 WARN org.apache.hadoop.mapred.TaskTracker: 
>>> getMapOutput(attempt_201208301039_0001_m_000204_0,0) failed :
>>> org.mortbay.jetty.EofException
>>>     at org.mortbay.jetty.HttpGenerator.flush(HttpGenerator.java:791)
>>>     at 
>>> org.mortbay.jetty.AbstractGenerator$Output.blockForOutput(AbstractGenerator.java:551)
>>>     at 
>>> org.mortbay.jetty.AbstractGenerator$Output.flush(AbstractGenerator.java:572)
>>>     at 
>>> org.mortbay.jetty.HttpConnection$Output.flush(HttpConnection.java:1012)
>>>     at 
>>> org.mortbay.jetty.AbstractGenerator$Output.write(AbstractGenerator.java:651)
>>>     at 
>>> org.mortbay.jetty.AbstractGenerator$Output.write(AbstractGenerator.java:580)
>>>     at 
>>> org.apache.hadoop.mapred.TaskTracker$MapOutputServlet.doGet(TaskTracker.java:3916)
>>>     at javax.servlet.http.HttpServlet.service(HttpServlet.java:707)
>>>     at javax.servlet.http.HttpServlet.service(HttpServlet.java:820)
>>>     at 
>>> org.mortbay.jetty.servlet.ServletHolder.handle(ServletHolder.java:511)
>>>     at 
>>> org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1221)
>>>     at 
>>> org.apache.hadoop.http.HttpServer$QuotingInputFilter.doFilter(HttpServer.java:835)
>>>     at 
>>> org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212)
>>>     at 
>>> org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:399) 
>>>
>>>     at 
>>> org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:216)
>>>     at 
>>> org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:182) 
>>>
>>>     at 
>>> org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766) 
>>>
>>>     at 
>>> org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:450)
>>>     at 
>>> org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:230)
>>>     at 
>>> org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152) 
>>>
>>>     at org.mortbay.jetty.Server.handle(Server.java:326)
>>>     at 
>>> org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:542)
>>>     at 
>>> org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConnection.java:928)
>>>     at org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:549)
>>>     at org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:212)
>>>     at org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:404)
>>>     at 
>>> org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:410)
>>>     at 
>>> org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582)
>>> Caused by: java.io.IOException: Broken pipe
>>>     at sun.nio.ch.FileDispatcherImpl.write0(Native Method)
>>>     at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:47)
>>>     at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:89)
>>>     at sun.nio.ch.IOUtil.write(IOUtil.java:60)
>>>     at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:450)
>>>     at 
>>> org.mortbay.io.nio.ChannelEndPoint.flush(ChannelEndPoint.java:170)
>>>     at 
>>> org.mortbay.io.nio.SelectChannelEndPoint.flush(SelectChannelEndPoint.java:221)
>>>     at org.mortbay.jetty.HttpGenerator.flush(HttpGenerator.java:725)
>>>     ... 27 more
>>>
>>> 2012-08-30 12:08:39,236 WARN org.mortbay.log: Committed before 410 
>>> getMapOutput(attempt_201208301039_0001_m_000204_0,0) failed :
>>> org.mortbay.jetty.EofException
>>>     at org.mortbay.jetty.HttpGenerator.flush(HttpGenerator.java:791)
>>>     at 
>>> org.mortbay.jetty.AbstractGenerator$Output.blockForOutput(AbstractGenerator.java:551)
>>>     at 
>>> org.mortbay.jetty.AbstractGenerator$Output.flush(AbstractGenerator.java:572)
>>>     at 
>>> org.mortbay.jetty.HttpConnection$Output.flush(HttpConnection.java:1012)
>>>     at 
>>> org.mortbay.jetty.AbstractGenerator$Output.write(AbstractGenerator.java:651)
>>>     at 
>>> org.mortbay.jetty.AbstractGenerator$Output.write(AbstractGenerator.java:580)
>>>     at 
>>> org.apache.hadoop.mapred.TaskTracker$MapOutputServlet.doGet(TaskTracker.java:3916)
>>>     at javax.servlet.http.HttpServlet.service(HttpServlet.java:707)
>>>     at javax.servlet.http.HttpServlet.service(HttpServlet.java:820)
>>>     at 
>>> org.mortbay.jetty.servlet.ServletHolder.handle(ServletHolder.java:511)
>>>     at 
>>> org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1221)
>>>     at 
>>> org.apache.hadoop.http.HttpServer$QuotingInputFilter.doFilter(HttpServer.java:835)
>>>     at 
>>> org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212)
>>>     at 
>>> org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:399) 
>>>
>>>     at 
>>> org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:216)
>>>     at 
>>> org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:182) 
>>>
>>>     at 
>>> org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766) 
>>>
>>>     at 
>>> org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:450)
>>>     at 
>>> org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:230)
>>>     at 
>>> org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152) 
>>>
>>>     at org.mortbay.jetty.Server.handle(Server.java:326)
>>>     at 
>>> org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:542)
>>>     at 
>>> org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConnection.java:928)
>>>     at org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:549)
>>>     at org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:212)
>>>     at org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:404)
>>>     at 
>>> org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:410)
>>>     at 
>>> org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582)
>>> Caused by: java.io.IOException: Broken pipe
>>>     at sun.nio.ch.FileDispatcherImpl.write0(Native Method)
>>>     at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:47)
>>>     at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:89)
>>>     at sun.nio.ch.IOUtil.write(IOUtil.java:60)
>>>     at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:450)
>>>     at 
>>> org.mortbay.io.nio.ChannelEndPoint.flush(ChannelEndPoint.java:170)
>>>     at 
>>> org.mortbay.io.nio.SelectChannelEndPoint.flush(SelectChannelEndPoint.java:221)
>>>     at org.mortbay.jetty.HttpGenerator.flush(HttpGenerator.java:725)
>>>     ... 27 more
>>>
>>> 2012-08-30 12:08:39,236 INFO 
>>> org.apache.hadoop.mapred.TaskTracker.clienttrace: src: 
>>> 141.51.205.130:35497, dest: 141.51.205.126:39614, bytes: 65536, op: 
>>> MAPRED_SHUFFLE, cliID: attempt_201208301039_0001_m_000204_0, 
>>> duration: 3209410
>>> 2012-08-30 12:08:39,237 ERROR org.mortbay.log: /mapOutput
>>> java.lang.IllegalStateException: Committed
>>>     at org.mortbay.jetty.Response.resetBuffer(Response.java:1023)
>>>     at org.mortbay.jetty.Response.sendError(Response.java:240)
>>>     at 
>>> org.apache.hadoop.mapred.TaskTracker$MapOutputServlet.doGet(TaskTracker.java:3945)
>>>     at javax.servlet.http.HttpServlet.service(HttpServlet.java:707)
>>>     at javax.servlet.http.HttpServlet.service(HttpServlet.java:820)
>>>     at 
>>> org.mortbay.jetty.servlet.ServletHolder.handle(ServletHolder.java:511)
>>>     at 
>>> org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1221)
>>>     at 
>>> org.apache.hadoop.http.HttpServer$QuotingInputFilter.doFilter(HttpServer.java:835)
>>>     at 
>>> org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212)
>>>     at 
>>> org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:399) 
>>>
>>>     at 
>>> org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:216)
>>>     at 
>>> org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:182) 
>>>
>>>     at 
>>> org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766) 
>>>
>>>     at 
>>> org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:450)
>>>     at 
>>> org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:230)
>>>     at 
>>> org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152) 
>>>
>>>     at org.mortbay.jetty.Server.handle(Server.java:326)
>>>     at 
>>> org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:542)
>>>     at 
>>> org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConnection.java:928)
>>>     at org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:549)
>>>     at org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:212)
>>>     at org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:404)
>>>     at 
>>> org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:410)
>>>     at 
>>> org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582)
>>> 2012-08-30 12:10:27,109 INFO 
>>> org.apache.hadoop.mapred.TaskTracker.clienttrace: src: 
>>> 141.51.205.130:35497, dest: 141.51.205.126:39618, bytes: 145730308, 
>>> op: MAPRED_SHUFFLE, cliID: attempt_201208301039_0001_m_000204_0, 
>>> duration: 5686096517
>>> 2012-08-30 12:14:29,739 WARN org.apache.hadoop.mapred.TaskTracker: 
>>> getMapOutput(attempt_201208301039_0001_m_000236_0,0) failed :
>>> org.mortbay.jetty.EofException
>>>     at org.mortbay.jetty.HttpGenerator.flush(HttpGenerator.java:791)
>>>     at 
>>> org.mortbay.jetty.AbstractGenerator$Output.blockForOutput(AbstractGenerator.java:551)
>>>     at 
>>> org.mortbay.jetty.AbstractGenerator$Output.flush(AbstractGenerator.java:572)
>>>     at 
>>> org.mortbay.jetty.HttpConnection$Output.flush(HttpConnection.java:1012)
>>>     at 
>>> org.mortbay.jetty.AbstractGenerator$Output.write(AbstractGenerator.java:651)
>>>     at 
>>> org.mortbay.jetty.AbstractGenerator$Output.write(AbstractGenerator.java:580)
>>>     at 
>>> org.apache.hadoop.mapred.TaskTracker$MapOutputServlet.doGet(TaskTracker.java:3916)
>>>     at javax.servlet.http.HttpServlet.service(HttpServlet.java:707)
>>>     at javax.servlet.http.HttpServlet.service(HttpServlet.java:820)
>>>     at 
>>> org.mortbay.jetty.servlet.ServletHolder.handle(ServletHolder.java:511)
>>>     at 
>>> org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1221)
>>>     at 
>>> org.apache.hadoop.http.HttpServer$QuotingInputFilter.doFilter(HttpServer.java:835)
>>>     at 
>>> org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212)
>>>     at 
>>> org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:399) 
>>>
>>>     at 
>>> org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:216)
>>>     at 
>>> org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:182) 
>>>
>>>     at 
>>> org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766) 
>>>
>>>     at 
>>> org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:450)
>>>     at 
>>> org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:230)
>>>     at 
>>> org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152) 
>>>
>>>     at org.mortbay.jetty.Server.handle(Server.java:326)
>>>     at 
>>> org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:542)
>>>     at 
>>> org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConnection.java:928)
>>>     at org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:549)
>>>     at org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:212)
>>>     at org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:404)
>>>     at 
>>> org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:410)
>>>     at 
>>> org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582)
>>> Caused by: java.io.IOException: Broken pipe
>>>     at sun.nio.ch.FileDispatcherImpl.write0(Native Method)
>>>     at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:47)
>>>     at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:89)
>>>     at sun.nio.ch.IOUtil.write(IOUtil.java:60)
>>>     at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:450)
>>>     at 
>>> org.mortbay.io.nio.ChannelEndPoint.flush(ChannelEndPoint.java:170)
>>>     at 
>>> org.mortbay.io.nio.SelectChannelEndPoint.flush(SelectChannelEndPoint.java:221)
>>>     at org.mortbay.jetty.HttpGenerator.flush(HttpGenerator.java:725)
>>>     ... 27 more
>>>
>>> 2012-08-30 12:14:29,740 WARN org.mortbay.log: Committed before 410 
>>> getMapOutput(attempt_201208301039_0001_m_000236_0,0) failed :
>>> org.mortbay.jetty.EofException
>>>     at org.mortbay.jetty.HttpGenerator.flush(HttpGenerator.java:791)
>>>     at 
>>> org.mortbay.jetty.AbstractGenerator$Output.blockForOutput(AbstractGenerator.java:551)
>>>     at 
>>> org.mortbay.jetty.AbstractGenerator$Output.flush(AbstractGenerator.java:572)
>>>     at 
>>> org.mortbay.jetty.HttpConnection$Output.flush(HttpConnection.java:1012)
>>>     at 
>>> org.mortbay.jetty.AbstractGenerator$Output.write(AbstractGenerator.java:651)
>>>     at 
>>> org.mortbay.jetty.AbstractGenerator$Output.write(AbstractGenerator.java:580)
>>>     at 
>>> org.apache.hadoop.mapred.TaskTracker$MapOutputServlet.doGet(TaskTracker.java:3916)
>>>     at javax.servlet.http.HttpServlet.service(HttpServlet.java:707)
>>>     at javax.servlet.http.HttpServlet.service(HttpServlet.java:820)
>>>     at 
>>> org.mortbay.jetty.servlet.ServletHolder.handle(ServletHolder.java:511)
>>>     at 
>>> org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1221)
>>>     at 
>>> org.apache.hadoop.http.HttpServer$QuotingInputFilter.doFilter(HttpServer.java:835)
>>>     at 
>>> org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212)
>>>     at 
>>> org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:399) 
>>>
>>>     at 
>>> org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:216)
>>>     at 
>>> org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:182) 
>>>
>>>     at 
>>> org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766) 
>>>
>>>     at 
>>> org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:450)
>>>     at 
>>> org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:230)
>>>     at 
>>> org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152) 
>>>
>>>     at org.mortbay.jetty.Server.handle(Server.java:326)
>>>     at 
>>> org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:542)
>>>     at 
>>> org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConnection.java:928)
>>>     at org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:549)
>>>     at org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:212)
>>>     at org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:404)
>>>     at 
>>> org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:410)
>>>     at 
>>> org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582)
>>> Caused by: java.io.IOException: Broken pipe
>>>     at sun.nio.ch.FileDispatcherImpl.write0(Native Method)
>>>     at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:47)
>>>     at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:89)
>>>     at sun.nio.ch.IOUtil.write(IOUtil.java:60)
>>>     at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:450)
>>>     at 
>>> org.mortbay.io.nio.ChannelEndPoint.flush(ChannelEndPoint.java:170)
>>>     at 
>>> org.mortbay.io.nio.SelectChannelEndPoint.flush(SelectChannelEndPoint.java:221)
>>>     at org.mortbay.jetty.HttpGenerator.flush(HttpGenerator.java:725)
>>>     ... 27 more
>>>
>>> 2012-08-30 12:14:29,740 INFO 
>>> org.apache.hadoop.mapred.TaskTracker.clienttrace: src: 
>>> 141.51.205.130:35497, dest: 141.51.205.126:39648, bytes: 65536, op: 
>>> MAPRED_SHUFFLE, cliID: attempt_201208301039_0001_m_000236_0, 
>>> duration: 3746212
>>> 2012-08-30 12:14:29,740 ERROR org.mortbay.log: /mapOutput
>>> java.lang.IllegalStateException: Committed
>>>     at org.mortbay.jetty.Response.resetBuffer(Response.java:1023)
>>>     at org.mortbay.jetty.Response.sendError(Response.java:240)
>>>     at 
>>> org.apache.hadoop.mapred.TaskTracker$MapOutputServlet.doGet(TaskTracker.java:3945)
>>>     at javax.servlet.http.HttpServlet.service(HttpServlet.java:707)
>>>     at javax.servlet.http.HttpServlet.service(HttpServlet.java:820)
>>>     at 
>>> org.mortbay.jetty.servlet.ServletHolder.handle(ServletHolder.java:511)
>>>     at 
>>> org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1221)
>>>     at 
>>> org.apache.hadoop.http.HttpServer$QuotingInputFilter.doFilter(HttpServer.java:835)
>>>     at 
>>> org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212)
>>>     at 
>>> org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:399) 
>>>
>>>     at 
>>> org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:216)
>>>     at 
>>> org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:182) 
>>>
>>>     at 
>>> org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766) 
>>>
>>>     at 
>>> org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:450)
>>>     at 
>>> org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:230)
>>>     at 
>>> org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152) 
>>>
>>>     at org.mortbay.jetty.Server.handle(Server.java:326)
>>>     at 
>>> org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:542)
>>>     at 
>>> org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConnection.java:928)
>>>     at org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:549)
>>>     at org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:212)
>>>     at org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:404)
>>>     at 
>>> org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:410)
>>>     at 
>>> org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582)
>>> 2012-08-30 12:16:28,503 INFO 
>>> org.apache.hadoop.mapred.TaskTracker.clienttrace: src: 
>>> 141.51.205.130:35497, dest: 141.51.205.126:39652, bytes: 145506420, 
>>> op: MAPRED_SHUFFLE, cliID: attempt_201208301039_0001_m_000236_0, 
>>> duration: 7793391393
>>> 2012-08-30 12:16:59,421 INFO org.mortbay.log: 
>>> org.mortbay.io.nio.SelectorManager$SelectSet@6bfd67ba JVM BUG(s) - 
>>> injecting delay1 times
>>> 2012-08-30 12:16:59,422 INFO org.mortbay.log: 
>>> org.mortbay.io.nio.SelectorManager$SelectSet@6bfd67ba JVM BUG(s) - 
>>> recreating selector 1 times, canceled keys 69 times
>>> 2012-08-30 12:20:01,203 WARN org.apache.hadoop.mapred.TaskTracker: 
>>> getMapOutput(attempt_201208301039_0001_m_000241_0,0) failed :
>>> org.mortbay.jetty.EofException
>>>     at org.mortbay.jetty.HttpGenerator.flush(HttpGenerator.java:791)
>>>     at 
>>> org.mortbay.jetty.AbstractGenerator$Output.blockForOutput(AbstractGenerator.java:551)
>>>     at 
>>> org.mortbay.jetty.AbstractGenerator$Output.flush(AbstractGenerator.java:572)
>>>     at 
>>> org.mortbay.jetty.HttpConnection$Output.flush(HttpConnection.java:1012)
>>>     at 
>>> org.mortbay.jetty.AbstractGenerator$Output.write(AbstractGenerator.java:651)
>>>     at 
>>> org.mortbay.jetty.AbstractGenerator$Output.write(AbstractGenerator.java:580)
>>>     at 
>>> org.apache.hadoop.mapred.TaskTracker$MapOutputServlet.doGet(TaskTracker.java:3916)
>>>     at javax.servlet.http.HttpServlet.service(HttpServlet.java:707)
>>>     at javax.servlet.http.HttpServlet.service(HttpServlet.java:820)
>>>     at 
>>> org.mortbay.jetty.servlet.ServletHolder.handle(ServletHolder.java:511)
>>>     at 
>>> org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1221)
>>>     at 
>>> org.apache.hadoop.http.HttpServer$QuotingInputFilter.doFilter(HttpServer.java:835)
>>>     at 
>>> org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212)
>>>     at 
>>> org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:399) 
>>>
>>>     at 
>>> org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:216)
>>>     at 
>>> org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:182) 
>>>
>>>     at 
>>> org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766) 
>>>
>>>     at 
>>> org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:450)
>>>     at 
>>> org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:230)
>>>     at 
>>> org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152) 
>>>
>>>     at org.mortbay.jetty.Server.handle(Server.java:326)
>>>     at 
>>> org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:542)
>>>     at 
>>> org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConnection.java:928)
>>>     at org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:549)
>>>     at org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:212)
>>>     at org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:404)
>>>     at 
>>> org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:410)
>>>     at 
>>> org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582)
>>> Caused by: java.io.IOException: Broken pipe
>>>     at sun.nio.ch.FileDispatcherImpl.write0(Native Method)
>>>     at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:47)
>>>     at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:89)
>>>     at sun.nio.ch.IOUtil.write(IOUtil.java:60)
>>>     at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:450)
>>>     at 
>>> org.mortbay.io.nio.ChannelEndPoint.flush(ChannelEndPoint.java:170)
>>>     at 
>>> org.mortbay.io.nio.SelectChannelEndPoint.flush(SelectChannelEndPoint.java:221)
>>>     at org.mortbay.jetty.HttpGenerator.flush(HttpGenerator.java:725)
>>>     ... 27 more
>>>
>>> 2012-08-30 12:20:01,205 WARN org.mortbay.log: Committed before 410 
>>> getMapOutput(attempt_201208301039_0001_m_000241_0,0) failed :
>>> org.mortbay.jetty.EofException
>>>     at org.mortbay.jetty.HttpGenerator.flush(HttpGenerator.java:791)
>>>     at 
>>> org.mortbay.jetty.AbstractGenerator$Output.blockForOutput(AbstractGenerator.java:551)
>>>     at 
>>> org.mortbay.jetty.AbstractGenerator$Output.flush(AbstractGenerator.java:572)
>>>     at 
>>> org.mortbay.jetty.HttpConnection$Output.flush(HttpConnection.java:1012)
>>>     at 
>>> org.mortbay.jetty.AbstractGenerator$Output.write(AbstractGenerator.java:651)
>>>     at 
>>> org.mortbay.jetty.AbstractGenerator$Output.write(AbstractGenerator.java:580)
>>>     at 
>>> org.apache.hadoop.mapred.TaskTracker$MapOutputServlet.doGet(TaskTracker.java:3916)
>>>     at javax.servlet.http.HttpServlet.service(HttpServlet.java:707)
>>>     at javax.servlet.http.HttpServlet.service(HttpServlet.java:820)
>>>     at 
>>> org.mortbay.jetty.servlet.ServletHolder.handle(ServletHolder.java:511)
>>>     at 
>>> org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1221)
>>>     at 
>>> org.apache.hadoop.http.HttpServer$QuotingInputFilter.doFilter(HttpServer.java:835)
>>>     at 
>>> org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212)
>>>     at 
>>> org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:399) 
>>>
>>>     at 
>>> org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:216)
>>>     at 
>>> org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:182) 
>>>
>>>     at 
>>> org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766) 
>>>
>>>     at 
>>> org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:450)
>>>     at 
>>> org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:230)
>>>     at 
>>> org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152) 
>>>
>>>     at org.mortbay.jetty.Server.handle(Server.java:326)
>>>     at 
>>> org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:542)
>>>     at 
>>> org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConnection.java:928)
>>>     at org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:549)
>>>     at org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:212)
>>>     at org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:404)
>>>     at 
>>> org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:410)
>>>     at 
>>> org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582)
>>> Caused by: java.io.IOException: Broken pipe
>>>     at sun.nio.ch.FileDispatcherImpl.write0(Native Method)
>>>     at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:47)
>>>     at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:89)
>>>     at sun.nio.ch.IOUtil.write(IOUtil.java:60)
>>>     at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:450)
>>>     at 
>>> org.mortbay.io.nio.ChannelEndPoint.flush(ChannelEndPoint.java:170)
>>>     at 
>>> org.mortbay.io.nio.SelectChannelEndPoint.flush(SelectChannelEndPoint.java:221)
>>>     at org.mortbay.jetty.HttpGenerator.flush(HttpGenerator.java:725)
>>>     ... 27 more
>>>
>>> 2012-08-30 12:20:01,205 INFO 
>>> org.apache.hadoop.mapred.TaskTracker.clienttrace: src: 
>>> 141.51.205.130:35497, dest: 141.51.205.126:39683, bytes: 65536, op: 
>>> MAPRED_SHUFFLE, cliID: attempt_201208301039_0001_m_000241_0, 
>>> duration: 5405031
>>> 2012-08-30 12:20:01,205 ERROR org.mortbay.log: /mapOutput
>>> java.lang.IllegalStateException: Committed
>>>     at org.mortbay.jetty.Response.resetBuffer(Response.java:1023)
>>>     at org.mortbay.jetty.Response.sendError(Response.java:240)
>>>     at 
>>> org.apache.hadoop.mapred.TaskTracker$MapOutputServlet.doGet(TaskTracker.java:3945)
>>>     at javax.servlet.http.HttpServlet.service(HttpServlet.java:707)
>>>     at javax.servlet.http.HttpServlet.service(HttpServlet.java:820)
>>>     at 
>>> org.mortbay.jetty.servlet.ServletHolder.handle(ServletHolder.java:511)
>>>     at 
>>> org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1221)
>>>     at 
>>> org.apache.hadoop.http.HttpServer$QuotingInputFilter.doFilter(HttpServer.java:835)
>>>     at 
>>> org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212)
>>>     at 
>>> org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:399) 
>>>
>>>     at 
>>> org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:216)
>>>     at 
>>> org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:182) 
>>>
>>>     at 
>>> org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766) 
>>>
>>>     at 
>>> org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:450)
>>>     at 
>>> org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:230)
>>>     at 
>>> org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152) 
>>>
>>>     at org.mortbay.jetty.Server.handle(Server.java:326)
>>>     at 
>>> org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:542)
>>>     at 
>>> org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConnection.java:928)
>>>     at org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:549)
>>>     at org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:212)
>>>     at org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:404)
>>>     at 
>>> org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:410)
>>>     at 
>>> org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582)
>>>
>>> Am 29.08.12 15:53, schrieb Björn-Elmar Macek:
>>>> Hi there,
>>>>
>>>> i am currently running a job where i selfjoin a 63 gigabyte big csv 
>>>> file on 20 physically distinct nodes with 15GB each:
>>>>
>>>> While the mapping works just fine and is low cost, the reducer does 
>>>> the main work: holding a hashmap with elements to join with and 
>>>> finding join tuples for evry incoming key-value-pair.
>>>>
>>>> The jobs works perfectly on small files with 2 gigabytes, but 
>>>> starts to get "unstable" as the file size goes up: this becomes 
>>>> evident with a look into the tasktracker's logs saying:
>>>>
>>>> ERROR org.mortbay.log: /mapOutput
>>>> java.lang.IllegalStateException: Committed
>>>>     at org.mortbay.jetty.Response.resetBuffer(Response.java:1023)
>>>>     at org.mortbay.jetty.Response.sendError(Response.java:240)
>>>>     at 
>>>> org.apache.hadoop.mapred.TaskTracker$MapOutputServlet.doGet(TaskTracker.java:3945)
>>>>     at javax.servlet.http.HttpServlet.service(HttpServlet.java:707)
>>>>     at javax.servlet.http.HttpServlet.service(HttpServlet.java:820)
>>>>     at 
>>>> org.mortbay.jetty.servlet.ServletHolder.handle(ServletHolder.java:511)
>>>>     at 
>>>> org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1221)
>>>>     at 
>>>> org.apache.hadoop.http.HttpServer$QuotingInputFilter.doFilter(HttpServer.java:835)
>>>>     at 
>>>> org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212)
>>>>     at 
>>>> org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:399) 
>>>>
>>>>     at 
>>>> org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:216)
>>>>     at 
>>>> org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:182) 
>>>>
>>>>     at 
>>>> org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766) 
>>>>
>>>>     at 
>>>> org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:450)
>>>>     at 
>>>> org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:230)
>>>>     at 
>>>> org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152) 
>>>>
>>>>     at org.mortbay.jetty.Server.handle(Server.java:326)
>>>>     at 
>>>> org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:542) 
>>>>
>>>>     at 
>>>> org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConnection.java:928)
>>>>     at org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:549)
>>>>     at 
>>>> org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:212)
>>>>     at 
>>>> org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:404)
>>>>     at 
>>>> org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:410)
>>>>     at 
>>>> org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582)
>>>>
>>>>
>>>> And while it is no problem at the beginning of the reduce process, 
>>>> where this happens only on a few nodes and rarely, it becomes 
>>>> crucial as the progress rises. The reason for this (afaik from 
>>>> reading articles), is that there are memory or file handle 
>>>> problems. I addressed the memory problem by conitiously purging the 
>>>> map of outdated elements evry 5 million processed key-value-pairs. 
>>>> And i set mapred.child.ulimit to 100000000 (ulimit in the shell 
>>>> tells me it is 400000000).
>>>>
>>>> Anyway i am still running into those mortbay errors and i start to 
>>>> wonder, if hadoop can manage the job with this algorithmn anyways. 
>>>> By pure naive math it should be:
>>>> i explicily assigned 10GB memory to each JVM on each node and set 
>>>> mapred.child.java.opts to "-Xmx10240m -XX:+UseCompressedOops 
>>>> -XX:-UseGCOverheadLimit" (its a 64 bit environment and large 
>>>> datastructures cause the GC to throw exceptions). This would 
>>>> naively make 18 slave machines with 10GB each resulting in an 
>>>> overall memory of 180GB - three times as much as needed... i would 
>>>> think. So if the Partitioner distributes them just about equally to 
>>>> all nodes i should not run into any errors, do i?
>>>>
>>>> Can anybody help me with this issue?
>>>>
>>>> Best regards,
>>>> Elmar
>>>>
>>>>
>>>
>>>
>>
>>
>
>


Mime
View raw message