tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From bi...@apache.org
Subject tez git commit: TEZ-2788. Allow TezAnalyzerBase to parse SimpleHistory logs (bikas)
Date Wed, 09 Sep 2015 01:50:56 GMT
Repository: tez
Updated Branches:
  refs/heads/master c07d6b751 -> 56f7847d2


TEZ-2788. Allow TezAnalyzerBase to parse SimpleHistory logs (bikas)


Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/56f7847d
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/56f7847d
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/56f7847d

Branch: refs/heads/master
Commit: 56f7847d26418879cfa32d7033147c8f95c56469
Parents: c07d6b7
Author: Bikas Saha <bikas@apache.org>
Authored: Tue Sep 8 18:50:47 2015 -0700
Committer: Bikas Saha <bikas@apache.org>
Committed: Tue Sep 8 18:50:47 2015 -0700

----------------------------------------------------------------------
 CHANGES.txt                                     |  1 +
 .../tez/analyzer/plugins/TezAnalyzerBase.java   | 63 +++++++++++++++++---
 2 files changed, 55 insertions(+), 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tez/blob/56f7847d/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 4717249..77535b1 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -7,6 +7,7 @@ Release 0.8.1: Unreleased
 INCOMPATIBLE CHANGES
 
 ALL CHANGES:
+  TEZ-2788. Allow TezAnalyzerBase to parse SimpleHistory logs
   TEZ-2782. VertexInfo.getAvgExecutionTimeInterval throws NPE when task does not have any
valid attempts info
   TEZ-2778. Improvements to handle multiple read errors with complex DAGs
   TEZ-2768. Log a useful error message when the summary stream cannot be closed when shutting

http://git-wip-us.apache.org/repos/asf/tez/blob/56f7847d/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/TezAnalyzerBase.java
----------------------------------------------------------------------
diff --git a/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/TezAnalyzerBase.java
b/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/TezAnalyzerBase.java
index 3eb2f57..fa3bd97 100644
--- a/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/TezAnalyzerBase.java
+++ b/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/TezAnalyzerBase.java
@@ -27,9 +27,12 @@ import org.apache.commons.cli.OptionBuilder;
 import org.apache.commons.cli.Options;
 import org.apache.commons.cli.ParseException;
 import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.util.Tool;
 import org.apache.tez.analyzer.Analyzer;
+import org.apache.tez.history.ATSImportTool;
 import org.apache.tez.history.parser.ATSFileParser;
+import org.apache.tez.history.parser.SimpleHistoryParser;
 import org.apache.tez.history.parser.datamodel.DagInfo;
 
 import com.google.common.base.Preconditions;
@@ -37,9 +40,10 @@ import com.google.common.base.Preconditions;
 public abstract class TezAnalyzerBase extends Configured implements Tool, Analyzer {
 
   
-  private static final String ATS_FILE_NAME = "atsFileName";
+  private static final String EVENT_FILE_NAME = "eventFileName";
   private static final String OUTPUT_DIR = "outputDir";
   private static final String DAG_ID = "dagId";
+  private static final String FROM_SIMPLE_HISTORY = "fromSimpleHistory";
   private static final String HELP = "help";
 
   private String outputDir;
@@ -52,10 +56,16 @@ public abstract class TezAnalyzerBase extends Configured implements Tool,
Analyz
     Option outputDirOption = OptionBuilder.withArgName(OUTPUT_DIR).withLongOpt(OUTPUT_DIR)
         .withDescription("Directory to write outputs to.").hasArg().isRequired(false).create();
 
-    Option inputATSFileNameOption = OptionBuilder.withArgName(ATS_FILE_NAME).withLongOpt
-        (ATS_FILE_NAME)
-        .withDescription("File with ATS data for the DAG").hasArg()
-        .isRequired(true).create();
+    Option eventFileNameOption = OptionBuilder.withArgName(EVENT_FILE_NAME).withLongOpt
+        (EVENT_FILE_NAME)
+        .withDescription("File with event data for the DAG").hasArg()
+        .isRequired(false).create();
+    
+    Option fromSimpleHistoryOption = OptionBuilder.withArgName(FROM_SIMPLE_HISTORY).withLongOpt
+        (FROM_SIMPLE_HISTORY)
+        .withDescription("Event data from Simple History logging. Must also specify event
file")
+        .isRequired(false).create();
+    
     Option help = OptionBuilder.withArgName(HELP).withLongOpt
         (HELP)
         .withDescription("print help")
@@ -64,7 +74,8 @@ public abstract class TezAnalyzerBase extends Configured implements Tool,
Analyz
     Options opts = new Options();
     opts.addOption(dagIdOption);
     opts.addOption(outputDirOption);
-    opts.addOption(inputATSFileNameOption);
+    opts.addOption(eventFileNameOption);
+    opts.addOption(fromSimpleHistoryOption);
     opts.addOption(help);
     return opts;
   }
@@ -103,12 +114,46 @@ public abstract class TezAnalyzerBase extends Configured implements
Tool, Analyz
     if (outputDir == null) {
       outputDir = System.getProperty("user.dir");
     }
+
+    File file = null;
+    if (cmdLine.hasOption(EVENT_FILE_NAME)) {
+      file = new File(cmdLine.getOptionValue(EVENT_FILE_NAME));
+    }
     
-    File file = new File(cmdLine.getOptionValue(ATS_FILE_NAME));
     String dagId = cmdLine.getOptionValue(DAG_ID);
     
-    ATSFileParser parser = new ATSFileParser(file);
-    DagInfo dagInfo = parser.getDAGData(dagId);
+    DagInfo dagInfo = null;
+    
+    if (file == null) {
+      if (cmdLine.hasOption(FROM_SIMPLE_HISTORY)) {
+        System.err.println("Event file name must be specified when using simple history");
+        printUsage();
+        return -2;
+      }
+      // using ATS - try to download directly
+      String[] importArgs = { "--dagId=" + dagId, "--downloadDir=" + outputDir };
+
+      int result = ATSImportTool.process(importArgs);
+      if (result != 0) {
+        System.err.println("Error downloading data from ATS");
+        return -3;
+      }
+
+      //Parse ATS data and verify results
+      //Parse downloaded contents
+      file = new File(outputDir
+          + Path.SEPARATOR + dagId
+          + Path.SEPARATOR + dagId + ".zip");
+    }
+    
+    Preconditions.checkState(file != null);
+    if (!cmdLine.hasOption(FROM_SIMPLE_HISTORY)) {
+      ATSFileParser parser = new ATSFileParser(file);
+      dagInfo = parser.getDAGData(dagId);
+    } else {
+      SimpleHistoryParser parser = new SimpleHistoryParser(file);
+      dagInfo = parser.getDAGData(dagId);
+    }
     Preconditions.checkState(dagInfo.getDagId().equals(dagId));
     analyze(dagInfo);
     return 0;


Mime
View raw message