Return-Path: Delivered-To: apmail-incubator-pig-commits-archive@locus.apache.org Received: (qmail 23595 invoked from network); 27 Sep 2008 00:00:55 -0000 Received: from hermes.apache.org (HELO mail.apache.org) (140.211.11.2) by minotaur.apache.org with SMTP; 27 Sep 2008 00:00:55 -0000 Received: (qmail 3578 invoked by uid 500); 27 Sep 2008 00:00:53 -0000 Delivered-To: apmail-incubator-pig-commits-archive@incubator.apache.org Received: (qmail 3559 invoked by uid 500); 27 Sep 2008 00:00:53 -0000 Mailing-List: contact pig-commits-help@incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: pig-dev@incubator.apache.org Delivered-To: mailing list pig-commits@incubator.apache.org Received: (qmail 3550 invoked by uid 99); 27 Sep 2008 00:00:53 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 26 Sep 2008 17:00:53 -0700 X-ASF-Spam-Status: No, hits=-2000.0 required=10.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 27 Sep 2008 00:00:00 +0000 Received: by eris.apache.org (Postfix, from userid 65534) id 4631723889B9; Fri, 26 Sep 2008 17:00:04 -0700 (PDT) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r699534 - in /incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer: Launcher.java LocalLauncher.java MapReduceLauncher.java Date: Sat, 27 Sep 2008 00:00:04 -0000 To: pig-commits@incubator.apache.org From: gates@apache.org X-Mailer: svnmailer-1.0.8 Message-Id: <20080927000004.4631723889B9@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: gates Date: Fri Sep 26 17:00:03 2008 New Revision: 699534 URL: http://svn.apache.org/viewvc?rev=699534&view=rev Log: PIG-457 This just addresses the issue of getting false error messages on success. Launchers changed to send any error messages from hadoop to debug if the job succeeded. Modified: incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/Launcher.java incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/LocalLauncher.java incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java Modified: incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/Launcher.java URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/Launcher.java?rev=699534&r1=699533&r2=699534&view=diff ============================================================================== --- incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/Launcher.java (original) +++ incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/Launcher.java Fri Sep 26 17:00:03 2008 @@ -85,13 +85,13 @@ return (int)(Math.ceil(prog)) == (int)1; } - protected void getStats(Job job, JobClient jobClient) throws IOException{ + protected void getStats(Job job, JobClient jobClient, boolean errNotDbg) throws IOException{ JobID MRJobID = job.getAssignedJobID(); TaskReport[] mapRep = jobClient.getMapTaskReports(MRJobID); - getErrorMessages(mapRep, "map"); + getErrorMessages(mapRep, "map", errNotDbg); totalHadoopTimeSpent += computeTimeSpent(mapRep); TaskReport[] redRep = jobClient.getReduceTaskReports(MRJobID); - getErrorMessages(redRep, "reduce"); + getErrorMessages(redRep, "reduce", errNotDbg); totalHadoopTimeSpent += computeTimeSpent(mapRep); } @@ -103,13 +103,18 @@ return timeSpent; } - protected void getErrorMessages(TaskReport reports[], String type) + protected void getErrorMessages(TaskReport reports[], String type, boolean errNotDbg) { for (int i = 0; i < reports.length; i++) { String msgs[] = reports[i].getDiagnostics(); for (int j = 0; j < msgs.length; j++) { - log.error("Error message from task (" + type + ") " + - reports[i].getTaskID() + msgs[j]); + if (errNotDbg) { + log.error("Error message from task (" + type + ") " + + reports[i].getTaskID() + msgs[j]); + } else { + log.debug("Error message from task (" + type + ") " + + reports[i].getTaskID() + msgs[j]); + } } } } Modified: incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/LocalLauncher.java URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/LocalLauncher.java?rev=699534&r1=699533&r2=699534&view=diff ============================================================================== --- incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/LocalLauncher.java (original) +++ incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/LocalLauncher.java Fri Sep 26 17:00:03 2008 @@ -20,6 +20,7 @@ import org.apache.pig.backend.hadoop.datastorage.ConfigurationUtil; import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.plans.MROperPlan; import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.plans.MRPrinter; +import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.plans.MRStreamHandler; import org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan; import org.apache.pig.impl.plan.PlanException; import org.apache.pig.impl.plan.VisitorException; @@ -70,15 +71,16 @@ log.error("Map reduce job failed"); for (Job fj : failedJobs) { log.error(fj.getMessage()); - getStats(fj, jobClient); + getStats(fj, jobClient, true); } + jc.stop(); return false; } List succJobs = jc.getSuccessfulJobs(); if(succJobs!=null) for(Job job : succJobs){ - getStats(job,jobClient); + getStats(job,jobClient, false); } jc.stop(); @@ -111,6 +113,11 @@ CombinerOptimizer co = new CombinerOptimizer(plan); co.visit(); } + + // check whether stream operator is present + MRStreamHandler checker = new MRStreamHandler(plan); + checker.visit(); + // figure out the type of the key for the map plan // this is needed when the key is null to create // an appropriate NullableXXXWritable object @@ -168,13 +175,13 @@ throw new ExecException( "Something terribly wrong with Job Control."); for (Job job : failedJobs) { - getStats(job, jobClient); + getStats(job, jobClient, true); } } List succJobs = jc.getSuccessfulJobs(); if (succJobs != null) for (Job job : succJobs) { - getStats(job, jobClient); + getStats(job, jobClient, false); } jc.stop(); Modified: incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java?rev=699534&r1=699533&r2=699534&view=diff ============================================================================== --- incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java (original) +++ incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java Fri Sep 26 17:00:03 2008 @@ -88,15 +88,16 @@ log.error("Map reduce job failed"); for (Job fj : failedJobs) { log.error(fj.getMessage()); - getStats(fj, jobClient); + getStats(fj, jobClient, true); } + jc.stop(); return false; } List succJobs = jc.getSuccessfulJobs(); if(succJobs!=null) for(Job job : succJobs){ - getStats(job,jobClient); + getStats(job,jobClient, false); } jc.stop();