tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From rbalamo...@apache.org
Subject tez git commit: TEZ-3327. ATS Parser: Populate config details available in dag (rbalamohan)
Date Thu, 07 Jul 2016 22:24:47 GMT
Repository: tez
Updated Branches:
  refs/heads/master ccd0e04fe -> bcdef6605


TEZ-3327. ATS Parser: Populate config details available in dag (rbalamohan)


Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/bcdef660
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/bcdef660
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/bcdef660

Branch: refs/heads/master
Commit: bcdef660558d41b8f466417b52a08b037ba1e4a5
Parents: ccd0e04
Author: Rajesh Balamohan <rbalamohan@apache.org>
Authored: Fri Jul 8 03:54:31 2016 +0530
Committer: Rajesh Balamohan <rbalamohan@apache.org>
Committed: Fri Jul 8 03:54:31 2016 +0530

----------------------------------------------------------------------
 CHANGES.txt                                            |  1 +
 .../org/apache/tez/history/parser/ATSFileParser.java   | 12 +++++++++++-
 .../tez/history/parser/datamodel/BaseParser.java       |  3 +++
 .../apache/tez/history/parser/datamodel/DagInfo.java   |  9 +++++++++
 .../java/org/apache/tez/history/TestHistoryParser.java | 13 +++++++++++++
 5 files changed, 37 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tez/blob/bcdef660/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index c973ad8..02334b4 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -7,6 +7,7 @@ INCOMPATIBLE CHANGES
 
 ALL CHANGES:
 
+  TEZ-3327. ATS Parser: Populate config details available in dag.
   TEZ-3325. Flaky test in TestDAGImpl.testCounterLimits.
   TEZ-3323. Update licese and notice for xml-apis jar. Also update year in notice to 2016.
   TEZ-3223. Support a NullHistoryLogger to disable history logging if needed.

http://git-wip-us.apache.org/repos/asf/tez/blob/bcdef660/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/ATSFileParser.java
----------------------------------------------------------------------
diff --git a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/ATSFileParser.java
b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/ATSFileParser.java
index a789536..89db976 100644
--- a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/ATSFileParser.java
+++ b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/ATSFileParser.java
@@ -19,6 +19,7 @@
 package org.apache.tez.history.parser;
 
 import com.google.common.base.Preconditions;
+import com.google.common.collect.Maps;
 import org.apache.commons.io.IOUtils;
 import org.apache.tez.dag.api.TezException;
 import org.apache.tez.history.parser.datamodel.BaseParser;
@@ -39,6 +40,7 @@ import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
 import java.util.Enumeration;
+import java.util.Iterator;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipFile;
 
@@ -158,7 +160,15 @@ public class ATSFileParser extends BaseParser implements ATSData {
           String revision = tezVersion.optString(Constants.REVISION);
           this.versionInfo = new VersionInfo(version, revision, buildTime);
         }
-        //TODO: might need to parse config info? (e.g, hive settings etc. could consume memory)
+        //Parse Config info
+        this.config = Maps.newHashMap();
+        JSONObject configNode = otherInfoNode.getJSONObject(Constants.CONFIG);
+        Iterator it = configNode.keys();
+        while(it.hasNext()) {
+          String key = (String) it.next();
+          String value = configNode.getString(key);
+          config.put(key, value);
+        }
       }
       LOG.debug("Finished parsing tez application");
     }

http://git-wip-us.apache.org/repos/asf/tez/blob/bcdef660/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/BaseParser.java
----------------------------------------------------------------------
diff --git a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/BaseParser.java
b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/BaseParser.java
index 84fd976..362dbd9 100644
--- a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/BaseParser.java
+++ b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/BaseParser.java
@@ -26,11 +26,13 @@ import org.apache.tez.dag.records.TezTaskAttemptID;
 import org.apache.tez.dag.records.TezTaskID;
 
 import java.util.List;
+import java.util.Map;
 
 public abstract class BaseParser {
 
   protected DagInfo dagInfo;
   protected VersionInfo versionInfo;
+  protected Map<String, String> config;
   protected final List<VertexInfo> vertexList;
   protected final List<TaskInfo> taskList;
   protected final List<TaskAttemptInfo> attemptList;
@@ -95,6 +97,7 @@ public abstract class BaseParser {
     }
 
     dagInfo.setVersionInfo(versionInfo);
+    dagInfo.setAppConfig(config);
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/tez/blob/bcdef660/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/DagInfo.java
----------------------------------------------------------------------
diff --git a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/DagInfo.java
b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/DagInfo.java
index 228a159..6bd691c 100644
--- a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/DagInfo.java
+++ b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/DagInfo.java
@@ -83,6 +83,7 @@ public class DagInfo extends BaseInfo {
   private Map<String, VertexInfo> vertexNameMap;
 
   private Multimap<Container, TaskAttemptInfo> containerMapping;
+  private Map<String, String> config;
 
   DagInfo(JSONObject jsonObject) throws JSONException {
     super(jsonObject);
@@ -350,6 +351,14 @@ public class DagInfo extends BaseInfo {
     vertexNameMap.put(vertexInfo.getVertexName(), vertexInfo);
   }
 
+  void setAppConfig(Map<String, String> config) {
+    this.config = config;
+  }
+
+  public Map<String, String> getAppConfig() {
+    return (config != null) ? Collections.unmodifiableMap(config) : null;
+  }
+
   void setVersionInfo(VersionInfo versionInfo) {
     this.versionInfo = versionInfo;
   }

http://git-wip-us.apache.org/repos/asf/tez/blob/bcdef660/tez-plugins/tez-history-parser/src/test/java/org/apache/tez/history/TestHistoryParser.java
----------------------------------------------------------------------
diff --git a/tez-plugins/tez-history-parser/src/test/java/org/apache/tez/history/TestHistoryParser.java
b/tez-plugins/tez-history-parser/src/test/java/org/apache/tez/history/TestHistoryParser.java
index 401a606..92c4ad8 100644
--- a/tez-plugins/tez-history-parser/src/test/java/org/apache/tez/history/TestHistoryParser.java
+++ b/tez-plugins/tez-history-parser/src/test/java/org/apache/tez/history/TestHistoryParser.java
@@ -210,6 +210,7 @@ public class TestHistoryParser {
     DagInfo dagInfoFromATS = getDagInfo(dagId);
     verifyDagInfo(dagInfoFromATS, true);
     verifyJobSpecificInfo(dagInfoFromATS);
+    checkConfig(dagInfoFromATS);
 
     //Now run with SimpleHistoryLogging
     dagId = runWordCount(WordCount.TokenProcessor.class.getName(),
@@ -244,6 +245,16 @@ public class TestHistoryParser {
     return dagInfo;
   }
 
+  private void checkConfig(DagInfo dagInfo) {
+    assertTrue("DagInfo is " + dagInfo, dagInfo != null);
+    //Check configs
+    assertTrue("DagInfo config size=" + dagInfo.getAppConfig().size(),
+        dagInfo.getAppConfig().size() > 0);
+    //Sample config element
+    assertTrue("DagInfo config=" + dagInfo.getAppConfig(),
+        Integer.parseInt(dagInfo.getAppConfig().get("dfs.replication")) > 0);
+  }
+
   private void verifyJobSpecificInfo(DagInfo dagInfo) {
     //Job specific
     assertTrue(dagInfo.getNumVertices() == 2);
@@ -373,6 +384,8 @@ public class TestHistoryParser {
 
     //Parse ATS data
     DagInfo dagInfo = getDagInfo(dagId);
+    //Applicable for ATS dataset
+    checkConfig(dagInfo);
 
     //Verify DAGInfo. Verifies vertex, task, taskAttempts in recursive manner
     verifyDagInfo(dagInfo, true);


Mime
View raw message