hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hashut...@apache.org
Subject svn commit: r1408803 - in /hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec: HadoopJobExecHelper.java JobDebugger.java
Date Tue, 13 Nov 2012 15:55:23 GMT
Author: hashutosh
Date: Tue Nov 13 15:55:23 2012
New Revision: 1408803

URL: http://svn.apache.org/viewvc?rev=1408803&view=rev
Log:
HIVE-3215 : JobDebugger should use RunningJob.getTrackingURL (Bhushan Mandhani via Ashutosh
Chauhan)

Modified:
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/HadoopJobExecHelper.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/JobDebugger.java

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/HadoopJobExecHelper.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/HadoopJobExecHelper.java?rev=1408803&r1=1408802&r2=1408803&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/HadoopJobExecHelper.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/HadoopJobExecHelper.java Tue Nov
13 15:55:23 2012
@@ -20,18 +20,14 @@ package org.apache.hadoop.hive.ql.exec;
 
 import java.io.IOException;
 import java.io.Serializable;
-import java.lang.Exception;
-import java.net.MalformedURLException;
 import java.text.SimpleDateFormat;
 import java.util.ArrayList;
 import java.util.Calendar;
 import java.util.Collections;
 import java.util.Enumeration;
 import java.util.HashMap;
-import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
-import java.util.Set;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -40,8 +36,6 @@ import org.apache.hadoop.hive.common.Jav
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.ql.MapRedStats;
 import org.apache.hadoop.hive.ql.exec.Operator.ProgressCounter;
-import org.apache.hadoop.hive.ql.exec.errors.ErrorAndSolution;
-import org.apache.hadoop.hive.ql.exec.errors.TaskLogProcessor;
 import org.apache.hadoop.hive.ql.history.HiveHistory.Keys;
 import org.apache.hadoop.hive.ql.plan.ReducerTimeStatsPerJob;
 import org.apache.hadoop.hive.ql.session.SessionState;
@@ -470,165 +464,6 @@ public class HadoopJobExecHelper {
     }
   }
 
-  // Used for showJobFailDebugInfo
-  private static class TaskInfo {
-    String jobId;
-    HashSet<String> logUrls;
-
-    public TaskInfo(String jobId) {
-      this.jobId = jobId;
-      logUrls = new HashSet<String>();
-    }
-
-    public void addLogUrl(String logUrl) {
-      logUrls.add(logUrl);
-    }
-
-    public HashSet<String> getLogUrls() {
-      return logUrls;
-    }
-
-    public String getJobId() {
-      return jobId;
-    }
-  }
-
-  @SuppressWarnings("deprecation")
-  private void showJobFailDebugInfo(JobConf conf, RunningJob rj)
-    throws IOException, MalformedURLException {
-    // Mapping from task ID to the number of failures
-    Map<String, Integer> failures = new HashMap<String, Integer>();
-    // Successful task ID's
-    Set<String> successes = new HashSet<String>();
-
-    Map<String, TaskInfo> taskIdToInfo = new HashMap<String, TaskInfo>();
-
-    int startIndex = 0;
-
-    console.printError("Error during job, obtaining debugging information...");
-    // Loop to get all task completion events because getTaskCompletionEvents
-    // only returns a subset per call
-    while (true) {
-      TaskCompletionEvent[] taskCompletions = rj.getTaskCompletionEvents(startIndex);
-
-      if (taskCompletions == null || taskCompletions.length == 0) {
-        break;
-      }
-
-      boolean more = true;
-      boolean firstError = true;
-      for (TaskCompletionEvent t : taskCompletions) {
-        // getTaskJobIDs returns Strings for compatibility with Hadoop versions
-        // without TaskID or TaskAttemptID
-        String[] taskJobIds = ShimLoader.getHadoopShims().getTaskJobIDs(t);
-
-        if (taskJobIds == null) {
-          console.printError("Task attempt info is unavailable in this Hadoop version");
-          more = false;
-          break;
-        }
-
-        // For each task completion event, get the associated task id, job id
-        // and the logs
-        String taskId = taskJobIds[0];
-        String jobId = taskJobIds[1];
-        if (firstError) {
-          console.printError("Examining task ID: " + taskId + " (and more) from job " + jobId);
-          firstError = false;
-        }
-
-        TaskInfo ti = taskIdToInfo.get(taskId);
-        if (ti == null) {
-          ti = new TaskInfo(jobId);
-          taskIdToInfo.put(taskId, ti);
-        }
-        // These tasks should have come from the same job.
-        assert (ti.getJobId() != null && ti.getJobId().equals(jobId));
-        String taskAttemptLogUrl = ShimLoader.getHadoopShims().getTaskAttemptLogUrl(
-          conf, t.getTaskTrackerHttp(), t.getTaskId());
-        if (taskAttemptLogUrl != null) {
-          ti.getLogUrls().add(taskAttemptLogUrl);
-        }
-
-        // If a task failed, then keep track of the total number of failures
-        // for that task (typically, a task gets re-run up to 4 times if it
-        // fails
-
-        if (t.getTaskStatus() != TaskCompletionEvent.Status.SUCCEEDED) {
-          Integer failAttempts = failures.get(taskId);
-          if (failAttempts == null) {
-            failAttempts = Integer.valueOf(0);
-          }
-          failAttempts = Integer.valueOf(failAttempts.intValue() + 1);
-          failures.put(taskId, failAttempts);
-        } else {
-          successes.add(taskId);
-        }
-      }
-      if (!more) {
-        break;
-      }
-      startIndex += taskCompletions.length;
-    }
-    // Remove failures for tasks that succeeded
-    for (String task : successes) {
-      failures.remove(task);
-    }
-
-    if (failures.keySet().size() == 0) {
-      return;
-    }
-
-    // Find the highest failure count
-    int maxFailures = 0;
-    for (Integer failCount : failures.values()) {
-      if (maxFailures < failCount.intValue()) {
-        maxFailures = failCount.intValue();
-      }
-    }
-
-    // Display Error Message for tasks with the highest failure count
-    String jtUrl = JobTrackerURLResolver.getURL(conf);
-
-    for (String task : failures.keySet()) {
-      if (failures.get(task).intValue() == maxFailures) {
-        TaskInfo ti = taskIdToInfo.get(task);
-        String jobId = ti.getJobId();
-        String taskUrl = jtUrl + "/taskdetails.jsp?jobid=" + jobId + "&tipid=" + task.toString();
-
-        TaskLogProcessor tlp = new TaskLogProcessor(conf);
-        for (String logUrl : ti.getLogUrls()) {
-          tlp.addTaskAttemptLogUrl(logUrl);
-        }
-
-        List<ErrorAndSolution> errors = tlp.getErrors();
-
-        StringBuilder sb = new StringBuilder();
-        // We use a StringBuilder and then call printError only once as
-        // printError will write to both stderr and the error log file. In
-        // situations where both the stderr and the log file output is
-        // simultaneously output to a single stream, this will look cleaner.
-        sb.append("\n");
-        sb.append("Task with the most failures(" + maxFailures + "): \n");
-        sb.append("-----\n");
-        sb.append("Task ID:\n  " + task + "\n\n");
-        sb.append("URL:\n  " + taskUrl + "\n");
-
-        for (ErrorAndSolution e : errors) {
-          sb.append("\n");
-          sb.append("Possible error:\n  " + e.getError() + "\n\n");
-          sb.append("Solution:\n  " + e.getSolution() + "\n");
-        }
-        sb.append("-----\n");
-
-        console.printError(sb.toString());
-        // Only print out one task because that's good enough for debugging.
-        break;
-      }
-    }
-    return;
-
-  }
 
   public void localJobDebugger(int exitVal, String taskId) {
     StringBuilder sb = new StringBuilder();

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/JobDebugger.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/JobDebugger.java?rev=1408803&r1=1408802&r2=1408803&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/JobDebugger.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/JobDebugger.java Tue Nov 13 15:55:23
2012
@@ -193,10 +193,10 @@ public class JobDebugger implements Runn
           // Also keep track of the total number of failures for that
           // task (typically, a task gets re-run up to 4 times if it fails.
           if (t.getTaskStatus() != TaskCompletionEvent.Status.SUCCEEDED) {
+            String[] diags = rj.getTaskDiagnostics(t.getTaskAttemptId());
+            ti.setDiagnosticMesgs(diags);
             if (ti.getErrorCode() == 0) {
-              String[] diags = rj.getTaskDiagnostics(t.getTaskAttemptId());
               ti.setErrorCode(extractErrorCode(diags));
-              ti.setDiagnosticMesgs(diags);
             }
 
             Integer failAttempts = failures.get(taskId);
@@ -229,6 +229,10 @@ public class JobDebugger implements Runn
   @SuppressWarnings("deprecation")
   private void showJobFailDebugInfo() throws IOException {
     console.printError("Error during job, obtaining debugging information...");
+    if (!conf.get("mapred.job.tracker", "local").equals("local")) {
+      // Show Tracking URL for remotely running jobs.
+      console.printError("Job Tracking URL: " + rj.getTrackingURL());
+    }
     // Loop to get all task completion events because getTaskCompletionEvents
     // only returns a subset per call
     TaskInfoGrabber tlg = new TaskInfoGrabber();
@@ -265,8 +269,8 @@ public class JobDebugger implements Runn
       if (failures.get(task).intValue() == maxFailures) {
         TaskInfo ti = taskIdToInfo.get(task);
         String jobId = ti.getJobId();
-        String taskUrl = (jtUrl == null) ? "Unavailable" :
-            jtUrl + "/taskdetails.jsp?jobid=" + jobId + "&tipid=" + task.toString();
+        String taskUrl = (jtUrl == null) ? null :
+          jtUrl + "/taskdetails.jsp?jobid=" + jobId + "&tipid=" + task.toString();
 
         TaskLogProcessor tlp = new TaskLogProcessor(conf);
         for (String logUrl : ti.getLogUrls()) {
@@ -293,7 +297,9 @@ public class JobDebugger implements Runn
           sb.append("Task with the most failures(" + maxFailures + "): \n");
           sb.append("-----\n");
           sb.append("Task ID:\n  " + task + "\n\n");
-          sb.append("URL:\n  " + taskUrl + "\n");
+          if (taskUrl != null) {
+            sb.append("URL:\n  " + taskUrl + "\n");
+          }
 
           for (ErrorAndSolution e : errors) {
             sb.append("\n");



Mime
View raw message