pig-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ga...@apache.org
Subject svn commit: r699534 - in /incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer: Launcher.java LocalLauncher.java MapReduceLauncher.java
Date Sat, 27 Sep 2008 00:00:04 GMT
Author: gates
Date: Fri Sep 26 17:00:03 2008
New Revision: 699534

URL: http://svn.apache.org/viewvc?rev=699534&view=rev
Log:
PIG-457 This just addresses the issue of getting false error messages on success.  Launchers
changed to send any error messages from hadoop to debug if the
job succeeded.


Modified:
    incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/Launcher.java
    incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/LocalLauncher.java
    incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java

Modified: incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/Launcher.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/Launcher.java?rev=699534&r1=699533&r2=699534&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/Launcher.java
(original)
+++ incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/Launcher.java
Fri Sep 26 17:00:03 2008
@@ -85,13 +85,13 @@
         return (int)(Math.ceil(prog)) == (int)1;
     }
     
-    protected void getStats(Job job, JobClient jobClient) throws IOException{
+    protected void getStats(Job job, JobClient jobClient, boolean errNotDbg) throws IOException{
         JobID MRJobID = job.getAssignedJobID();
         TaskReport[] mapRep = jobClient.getMapTaskReports(MRJobID);
-        getErrorMessages(mapRep, "map");
+        getErrorMessages(mapRep, "map", errNotDbg);
         totalHadoopTimeSpent += computeTimeSpent(mapRep);
         TaskReport[] redRep = jobClient.getReduceTaskReports(MRJobID);
-        getErrorMessages(redRep, "reduce");
+        getErrorMessages(redRep, "reduce", errNotDbg);
         totalHadoopTimeSpent += computeTimeSpent(mapRep);
     }
     
@@ -103,13 +103,18 @@
         return timeSpent;
     }
     
-    protected void getErrorMessages(TaskReport reports[], String type)
+    protected void getErrorMessages(TaskReport reports[], String type, boolean errNotDbg)
     {
         for (int i = 0; i < reports.length; i++) {
             String msgs[] = reports[i].getDiagnostics();
             for (int j = 0; j < msgs.length; j++) {
-                log.error("Error message from task (" + type + ") " +
-                    reports[i].getTaskID() + msgs[j]);
+                if (errNotDbg) {
+                    log.error("Error message from task (" + type + ") " +
+                        reports[i].getTaskID() + msgs[j]);
+                } else {
+                    log.debug("Error message from task (" + type + ") " +
+                        reports[i].getTaskID() + msgs[j]);
+                }
             }
         }
     }

Modified: incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/LocalLauncher.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/LocalLauncher.java?rev=699534&r1=699533&r2=699534&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/LocalLauncher.java
(original)
+++ incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/LocalLauncher.java
Fri Sep 26 17:00:03 2008
@@ -20,6 +20,7 @@
 import org.apache.pig.backend.hadoop.datastorage.ConfigurationUtil;
 import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.plans.MROperPlan;
 import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.plans.MRPrinter;
+import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.plans.MRStreamHandler;
 import org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan;
 import org.apache.pig.impl.plan.PlanException;
 import org.apache.pig.impl.plan.VisitorException;
@@ -70,15 +71,16 @@
             log.error("Map reduce job failed");
             for (Job fj : failedJobs) {
                 log.error(fj.getMessage());
-                getStats(fj, jobClient);
+                getStats(fj, jobClient, true);
             }
+            jc.stop(); 
             return false;
         }
 
         List<Job> succJobs = jc.getSuccessfulJobs();
         if(succJobs!=null)
             for(Job job : succJobs){
-                getStats(job,jobClient);
+                getStats(job,jobClient, false);
             }
 
         jc.stop(); 
@@ -111,6 +113,11 @@
             CombinerOptimizer co = new CombinerOptimizer(plan);
             co.visit();
         }
+
+        // check whether stream operator is present
+        MRStreamHandler checker = new MRStreamHandler(plan);
+        checker.visit();
+        
         // figure out the type of the key for the map plan
         // this is needed when the key is null to create
         // an appropriate NullableXXXWritable object
@@ -168,13 +175,13 @@
                 throw new ExecException(
                         "Something terribly wrong with Job Control.");
             for (Job job : failedJobs) {
-                getStats(job, jobClient);
+                getStats(job, jobClient, true);
             }
         }
         List<Job> succJobs = jc.getSuccessfulJobs();
         if (succJobs != null)
             for (Job job : succJobs) {
-                getStats(job, jobClient);
+                getStats(job, jobClient, false);
             }
 
         jc.stop();

Modified: incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java?rev=699534&r1=699533&r2=699534&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java
(original)
+++ incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java
Fri Sep 26 17:00:03 2008
@@ -88,15 +88,16 @@
             log.error("Map reduce job failed");
             for (Job fj : failedJobs) {
                 log.error(fj.getMessage());
-                getStats(fj, jobClient);
+                getStats(fj, jobClient, true);
             }
+            jc.stop(); 
             return false;
         }
 
         List<Job> succJobs = jc.getSuccessfulJobs();
         if(succJobs!=null)
             for(Job job : succJobs){
-                getStats(job,jobClient);
+                getStats(job,jobClient, false);
             }
 
         jc.stop(); 



Mime
View raw message