tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hit...@apache.org
Subject [4/4] tez git commit: TEZ-1931. Publish tez version info to Timeline. (hitesh)
Date Wed, 14 Jan 2015 03:05:31 GMT
TEZ-1931. Publish tez version info to Timeline. (hitesh)


Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/6bc500ff
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/6bc500ff
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/6bc500ff

Branch: refs/heads/master
Commit: 6bc500ff26795964e23870f24f489c43f864264f
Parents: 9fb863e
Author: Hitesh Shah <hitesh@apache.org>
Authored: Tue Jan 13 19:05:00 2015 -0800
Committer: Hitesh Shah <hitesh@apache.org>
Committed: Tue Jan 13 19:05:00 2015 -0800

----------------------------------------------------------------------
 .../org/apache/tez/common/ATSConstants.java     |  5 ++++
 .../java/org/apache/tez/common/VersionInfo.java | 14 ++++++++++
 .../org/apache/tez/dag/app/DAGAppMaster.java    |  3 ++-
 .../dag/history/events/AppLaunchedEvent.java    | 10 +++++--
 .../apache/tez/dag/history/utils/DAGUtils.java  | 10 +++++++
 .../TestHistoryEventsProtoConversion.java       |  2 +-
 .../impl/TestHistoryEventJsonConversion.java    |  2 +-
 .../ats/HistoryEventTimelineConversion.java     |  7 ++++-
 .../ats/TestHistoryEventTimelineConversion.java | 28 +++++++++++++++++---
 9 files changed, 71 insertions(+), 10 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tez/blob/6bc500ff/tez-api/src/main/java/org/apache/tez/common/ATSConstants.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/common/ATSConstants.java b/tez-api/src/main/java/org/apache/tez/common/ATSConstants.java
index e502374..26170f4 100644
--- a/tez-api/src/main/java/org/apache/tez/common/ATSConstants.java
+++ b/tez-api/src/main/java/org/apache/tez/common/ATSConstants.java
@@ -104,4 +104,9 @@ public class ATSConstants {
   public static final String DESCRIPTION = "desc";
   public static final String CONFIG = "config";
 
+  public static final String TEZ_VERSION = "tezVersion";
+  public static final String VERSION = "version";
+  public static final String REVISION = "revision";
+  public static final String BUILD_TIME = "buildTime";
+
 }

http://git-wip-us.apache.org/repos/asf/tez/blob/6bc500ff/tez-api/src/main/java/org/apache/tez/common/VersionInfo.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/common/VersionInfo.java b/tez-api/src/main/java/org/apache/tez/common/VersionInfo.java
index b5e44c9..7e2a7e5 100644
--- a/tez-api/src/main/java/org/apache/tez/common/VersionInfo.java
+++ b/tez-api/src/main/java/org/apache/tez/common/VersionInfo.java
@@ -29,6 +29,8 @@ import org.apache.hadoop.classification.InterfaceStability.Unstable;
 import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.util.ClassUtil;
 
+import com.google.common.annotations.VisibleForTesting;
+
 @Private
 @Unstable
 public class VersionInfo {
@@ -62,6 +64,18 @@ public class VersionInfo {
     }
   }
 
+  @VisibleForTesting
+  @Private
+  protected VersionInfo(String component, String version, String revision,
+      String buildTime, String scmUrl) {
+    this.info = new Properties();
+    this.component = component;
+    info.setProperty(VERSION, version);
+    info.setProperty(REVISION, revision);
+    info.setProperty(BUILD_TIME, buildTime);
+    info.setProperty(SCM_URL, scmUrl);
+  }
+
   public String getVersion() {
     return info.getProperty(VERSION, UNKNOWN);
   }

http://git-wip-us.apache.org/repos/asf/tez/blob/6bc500ff/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
index 30df120..04e3b1e 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
@@ -468,7 +468,8 @@ public class DAGAppMaster extends AbstractService {
     if (!versionMismatch) {
       if (this.appAttemptID.getAttemptId() == 1) {
         AppLaunchedEvent appLaunchedEvent = new AppLaunchedEvent(appAttemptID.getApplicationId(),
-            startTime, appSubmitTime, appMasterUgi.getShortUserName(), this.amConf);
+            startTime, appSubmitTime, appMasterUgi.getShortUserName(), this.amConf,
+            dagVersionInfo);
         historyEventHandler.handle(
             new DAGHistoryEvent(appLaunchedEvent));
       }

http://git-wip-us.apache.org/repos/asf/tez/blob/6bc500ff/tez-dag/src/main/java/org/apache/tez/dag/history/events/AppLaunchedEvent.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/AppLaunchedEvent.java
b/tez-dag/src/main/java/org/apache/tez/dag/history/events/AppLaunchedEvent.java
index 4c79c53..08d2aff 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/AppLaunchedEvent.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/AppLaunchedEvent.java
@@ -23,8 +23,8 @@ import java.io.InputStream;
 import java.io.OutputStream;
 
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.tez.common.VersionInfo;
 import org.apache.tez.dag.history.HistoryEvent;
 import org.apache.tez.dag.history.HistoryEventType;
 
@@ -35,18 +35,20 @@ public class AppLaunchedEvent implements HistoryEvent {
   private long appSubmitTime;
   private String user;
   private Configuration conf;
+  private VersionInfo version;
 
   public AppLaunchedEvent() {
   }
 
   public AppLaunchedEvent(ApplicationId appId,
       long launchTime, long appSubmitTime, String user,
-      Configuration conf) {
+      Configuration conf, VersionInfo version) {
     this.applicationId = appId;
     this.launchTime = launchTime;
     this.appSubmitTime = appSubmitTime;
     this.user = user;
     this.conf = conf;
+    this.version = version;
   }
 
   @Override
@@ -101,4 +103,8 @@ public class AppLaunchedEvent implements HistoryEvent {
     return conf;
   }
 
+  public VersionInfo getVersion() {
+    return version;
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/tez/blob/6bc500ff/tez-dag/src/main/java/org/apache/tez/dag/history/utils/DAGUtils.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/utils/DAGUtils.java b/tez-dag/src/main/java/org/apache/tez/dag/history/utils/DAGUtils.java
index 5f259989..0f34811 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/history/utils/DAGUtils.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/history/utils/DAGUtils.java
@@ -30,6 +30,7 @@ import java.util.TreeMap;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.tez.common.ATSConstants;
+import org.apache.tez.common.VersionInfo;
 import org.apache.tez.common.counters.CounterGroup;
 import org.apache.tez.common.counters.TezCounter;
 import org.apache.tez.common.counters.TezCounters;
@@ -371,4 +372,13 @@ public class DAGUtils {
     return atsConf;
   }
 
+  public static Map<String, Object> convertTezVersionToATSMap(VersionInfo versionInfo)
{
+    Map<String, Object> atsInfo = new TreeMap<String, Object>();
+    atsInfo.put(ATSConstants.VERSION, versionInfo.getVersion());
+    atsInfo.put(ATSConstants.BUILD_TIME, versionInfo.getBuildTime());
+    atsInfo.put(ATSConstants.REVISION, versionInfo.getRevision());
+    return atsInfo;
+  }
+
+
 }

http://git-wip-us.apache.org/repos/asf/tez/blob/6bc500ff/tez-dag/src/test/java/org/apache/tez/dag/history/events/TestHistoryEventsProtoConversion.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/history/events/TestHistoryEventsProtoConversion.java
b/tez-dag/src/test/java/org/apache/tez/dag/history/events/TestHistoryEventsProtoConversion.java
index fb1f6f9..2eb02c1 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/history/events/TestHistoryEventsProtoConversion.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/history/events/TestHistoryEventsProtoConversion.java
@@ -121,7 +121,7 @@ public class TestHistoryEventsProtoConversion {
 
   private void testAppLaunchedEvent() throws Exception {
     AppLaunchedEvent event = new AppLaunchedEvent(ApplicationId.newInstance(0, 1),
-        100, 100, null, new Configuration(false));
+        100, 100, null, new Configuration(false), null);
     try {
       testProtoConversion(event);
       fail("Expected to fail on conversion");

http://git-wip-us.apache.org/repos/asf/tez/blob/6bc500ff/tez-dag/src/test/java/org/apache/tez/dag/history/logging/impl/TestHistoryEventJsonConversion.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/history/logging/impl/TestHistoryEventJsonConversion.java
b/tez-dag/src/test/java/org/apache/tez/dag/history/logging/impl/TestHistoryEventJsonConversion.java
index a31cfca..0eeba0d 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/history/logging/impl/TestHistoryEventJsonConversion.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/history/logging/impl/TestHistoryEventJsonConversion.java
@@ -105,7 +105,7 @@ public class TestHistoryEventJsonConversion {
       switch (eventType) {
         case APP_LAUNCHED:
           event = new AppLaunchedEvent(applicationId, random.nextInt(), random.nextInt(),
-              user, new Configuration(false));
+              user, new Configuration(false), null);
           break;
         case AM_LAUNCHED:
           event = new AMLaunchedEvent(applicationAttemptId, random.nextInt(), random.nextInt(),

http://git-wip-us.apache.org/repos/asf/tez/blob/6bc500ff/tez-plugins/tez-yarn-timeline-history-with-acls/src/main/java/org/apache/tez/dag/history/logging/ats/HistoryEventTimelineConversion.java
----------------------------------------------------------------------
diff --git a/tez-plugins/tez-yarn-timeline-history-with-acls/src/main/java/org/apache/tez/dag/history/logging/ats/HistoryEventTimelineConversion.java
b/tez-plugins/tez-yarn-timeline-history-with-acls/src/main/java/org/apache/tez/dag/history/logging/ats/HistoryEventTimelineConversion.java
index 6dc3325..0c139ee 100644
--- a/tez-plugins/tez-yarn-timeline-history-with-acls/src/main/java/org/apache/tez/dag/history/logging/ats/HistoryEventTimelineConversion.java
+++ b/tez-plugins/tez-yarn-timeline-history-with-acls/src/main/java/org/apache/tez/dag/history/logging/ats/HistoryEventTimelineConversion.java
@@ -59,7 +59,7 @@ public class HistoryEventTimelineConversion {
       throw new UnsupportedOperationException("Invalid Event, does not support history"
           + ", eventType=" + historyEvent.getEventType());
     }
-    TimelineEntity timelineEntity = null;
+    TimelineEntity timelineEntity;
     switch (historyEvent.getEventType()) {
       case APP_LAUNCHED:
         timelineEntity = convertAppLaunchedEvent((AppLaunchedEvent) historyEvent);
@@ -144,6 +144,11 @@ public class HistoryEventTimelineConversion {
 
     atsEntity.setStartTime(event.getLaunchTime());
 
+    if (event.getVersion() != null) {
+      atsEntity.addOtherInfo(ATSConstants.TEZ_VERSION,
+          DAGUtils.convertTezVersionToATSMap(event.getVersion()));
+    }
+
     return atsEntity;
   }
 

http://git-wip-us.apache.org/repos/asf/tez/blob/6bc500ff/tez-plugins/tez-yarn-timeline-history-with-acls/src/test/java/org/apache/tez/dag/history/logging/ats/TestHistoryEventTimelineConversion.java
----------------------------------------------------------------------
diff --git a/tez-plugins/tez-yarn-timeline-history-with-acls/src/test/java/org/apache/tez/dag/history/logging/ats/TestHistoryEventTimelineConversion.java
b/tez-plugins/tez-yarn-timeline-history-with-acls/src/test/java/org/apache/tez/dag/history/logging/ats/TestHistoryEventTimelineConversion.java
index e1eb3a4..f357a52 100644
--- a/tez-plugins/tez-yarn-timeline-history-with-acls/src/test/java/org/apache/tez/dag/history/logging/ats/TestHistoryEventTimelineConversion.java
+++ b/tez-plugins/tez-yarn-timeline-history-with-acls/src/test/java/org/apache/tez/dag/history/logging/ats/TestHistoryEventTimelineConversion.java
@@ -30,6 +30,7 @@ import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
 import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent;
 import org.apache.tez.common.ATSConstants;
+import org.apache.tez.common.VersionInfo;
 import org.apache.tez.dag.api.EdgeManagerPluginDescriptor;
 import org.apache.tez.dag.api.oldrecords.TaskAttemptState;
 import org.apache.tez.dag.api.oldrecords.TaskState;
@@ -107,7 +108,7 @@ public class TestHistoryEventTimelineConversion {
       switch (eventType) {
         case APP_LAUNCHED:
           event = new AppLaunchedEvent(applicationId, random.nextInt(), random.nextInt(),
-              user, new Configuration(false));
+              user, new Configuration(false), null);
           break;
         case AM_LAUNCHED:
           event = new AMLaunchedEvent(applicationAttemptId, random.nextInt(), random.nextInt(),
@@ -192,6 +193,14 @@ public class TestHistoryEventTimelineConversion {
     }
   }
 
+  static class MockVersionInfo extends VersionInfo {
+
+    MockVersionInfo() {
+      super("component", "1.1.0", "rev1", "20120101", "git.apache.org");
+    }
+
+  }
+
   @Test(timeout = 5000)
   public void testConvertAppLaunchedEvent() {
     long launchTime = random.nextLong();
@@ -200,9 +209,9 @@ public class TestHistoryEventTimelineConversion {
     conf.set("foo", "bar");
     conf.set("applicationId", "1234");
 
-
+    MockVersionInfo mockVersionInfo = new MockVersionInfo();
     AppLaunchedEvent event = new AppLaunchedEvent(applicationId, launchTime,
-        submitTime, user, conf);
+        submitTime, user, conf, mockVersionInfo);
 
     TimelineEntity timelineEntity = HistoryEventTimelineConversion.convertToTimelineEntity(event);
 
@@ -220,13 +229,24 @@ public class TestHistoryEventTimelineConversion {
     Assert.assertEquals(1, timelineEntity.getPrimaryFilters().size());
     Assert.assertTrue(timelineEntity.getPrimaryFilters().get(ATSConstants.USER).contains(user));
 
-    Assert.assertEquals(1, timelineEntity.getOtherInfo().size());
+    Assert.assertEquals(2, timelineEntity.getOtherInfo().size());
     Assert.assertTrue(timelineEntity.getOtherInfo().containsKey(ATSConstants.CONFIG));
+    Assert.assertTrue(timelineEntity.getOtherInfo().containsKey(ATSConstants.TEZ_VERSION));
 
     Map<String, String> config =
         (Map<String, String>)timelineEntity.getOtherInfo().get(ATSConstants.CONFIG);
     Assert.assertEquals(conf.get("foo"), config.get("foo"));
     Assert.assertEquals(conf.get("applicationId"), config.get("applicationId"));
+
+    Map<String, String> versionInfo =
+        (Map<String, String>)timelineEntity.getOtherInfo().get(ATSConstants.TEZ_VERSION);
+    Assert.assertEquals(mockVersionInfo.getVersion(),
+        versionInfo.get(ATSConstants.VERSION));
+    Assert.assertEquals(mockVersionInfo.getRevision(),
+        versionInfo.get(ATSConstants.REVISION));
+    Assert.assertEquals(mockVersionInfo.getBuildTime(),
+        versionInfo.get(ATSConstants.BUILD_TIME));
+
   }
 
   @Test(timeout = 5000)


Mime
View raw message