tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hit...@apache.org
Subject tez git commit: TEZ-1931. Publish tez version info to Timeline. (hitesh)
Date Mon, 19 Jan 2015 22:39:04 GMT
Repository: tez
Updated Branches:
  refs/heads/branch-0.5 4bbd257db -> 6607e9448


TEZ-1931. Publish tez version info to Timeline. (hitesh)


Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/6607e944
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/6607e944
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/6607e944

Branch: refs/heads/branch-0.5
Commit: 6607e9448bae7dc24a777485dedb2e5cf787ee83
Parents: 4bbd257
Author: Hitesh Shah <hitesh@apache.org>
Authored: Mon Jan 19 14:38:36 2015 -0800
Committer: Hitesh Shah <hitesh@apache.org>
Committed: Mon Jan 19 14:38:36 2015 -0800

----------------------------------------------------------------------
 CHANGES.txt                                     |  1 +
 .../org/apache/tez/common/ATSConstants.java     |  6 +++++
 .../java/org/apache/tez/common/VersionInfo.java | 14 ++++++++++
 .../org/apache/tez/dag/app/DAGAppMaster.java    |  3 ++-
 .../dag/history/events/AppLaunchedEvent.java    | 10 +++++--
 .../apache/tez/dag/history/utils/DAGUtils.java  | 10 +++++++
 .../TestHistoryEventsProtoConversion.java       |  2 +-
 .../impl/TestHistoryEventJsonConversion.java    |  2 +-
 .../ats/HistoryEventTimelineConversion.java     |  7 ++++-
 .../ats/TestHistoryEventTimelineConversion.java | 28 +++++++++++++++++---
 10 files changed, 73 insertions(+), 10 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tez/blob/6607e944/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 7244db8..81ab0e2 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -4,6 +4,7 @@ Apache Tez Change Log
 Release 0.5.4: Unreleased
 
 ALL CHANGES:
+  TEZ-1931. Publish tez version info to Timeline.
   TEZ-1942. Number of tasks show in Tez UI with auto-reduce parallelism is misleading.
   TEZ-1962. Fix a thread leak in LocalMode.
   TEZ-1924. Tez AM does not register with AM with full FQDN causing jobs

http://git-wip-us.apache.org/repos/asf/tez/blob/6607e944/tez-api/src/main/java/org/apache/tez/common/ATSConstants.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/common/ATSConstants.java b/tez-api/src/main/java/org/apache/tez/common/ATSConstants.java
index 58761d5..aec6ca5 100644
--- a/tez-api/src/main/java/org/apache/tez/common/ATSConstants.java
+++ b/tez-api/src/main/java/org/apache/tez/common/ATSConstants.java
@@ -99,4 +99,10 @@ public class ATSConstants {
       "yarn.timeline-service.webapp.address";
   public static final String TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS_CONF_NAME =
       "yarn.timeline-service.webapp.https.address";
+
+  public static final String TEZ_VERSION = "tezVersion";
+  public static final String VERSION = "version";
+  public static final String REVISION = "revision";
+  public static final String BUILD_TIME = "buildTime";
+
 }

http://git-wip-us.apache.org/repos/asf/tez/blob/6607e944/tez-api/src/main/java/org/apache/tez/common/VersionInfo.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/common/VersionInfo.java b/tez-api/src/main/java/org/apache/tez/common/VersionInfo.java
index b5e44c9..7e2a7e5 100644
--- a/tez-api/src/main/java/org/apache/tez/common/VersionInfo.java
+++ b/tez-api/src/main/java/org/apache/tez/common/VersionInfo.java
@@ -29,6 +29,8 @@ import org.apache.hadoop.classification.InterfaceStability.Unstable;
 import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.util.ClassUtil;
 
+import com.google.common.annotations.VisibleForTesting;
+
 @Private
 @Unstable
 public class VersionInfo {
@@ -62,6 +64,18 @@ public class VersionInfo {
     }
   }
 
+  @VisibleForTesting
+  @Private
+  protected VersionInfo(String component, String version, String revision,
+      String buildTime, String scmUrl) {
+    this.info = new Properties();
+    this.component = component;
+    info.setProperty(VERSION, version);
+    info.setProperty(REVISION, revision);
+    info.setProperty(BUILD_TIME, buildTime);
+    info.setProperty(SCM_URL, scmUrl);
+  }
+
   public String getVersion() {
     return info.getProperty(VERSION, UNKNOWN);
   }

http://git-wip-us.apache.org/repos/asf/tez/blob/6607e944/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
index 789de24..c47d093 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
@@ -462,7 +462,8 @@ public class DAGAppMaster extends AbstractService {
     if (!versionMismatch) {
       if (this.appAttemptID.getAttemptId() == 1) {
         AppLaunchedEvent appLaunchedEvent = new AppLaunchedEvent(appAttemptID.getApplicationId(),
-            startTime, appSubmitTime, appMasterUgi.getShortUserName(), this.amConf);
+            startTime, appSubmitTime, appMasterUgi.getShortUserName(), this.amConf,
+            dagVersionInfo);
         historyEventHandler.handle(
             new DAGHistoryEvent(appLaunchedEvent));
       }

http://git-wip-us.apache.org/repos/asf/tez/blob/6607e944/tez-dag/src/main/java/org/apache/tez/dag/history/events/AppLaunchedEvent.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/AppLaunchedEvent.java
b/tez-dag/src/main/java/org/apache/tez/dag/history/events/AppLaunchedEvent.java
index 4c79c53..08d2aff 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/AppLaunchedEvent.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/AppLaunchedEvent.java
@@ -23,8 +23,8 @@ import java.io.InputStream;
 import java.io.OutputStream;
 
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.tez.common.VersionInfo;
 import org.apache.tez.dag.history.HistoryEvent;
 import org.apache.tez.dag.history.HistoryEventType;
 
@@ -35,18 +35,20 @@ public class AppLaunchedEvent implements HistoryEvent {
   private long appSubmitTime;
   private String user;
   private Configuration conf;
+  private VersionInfo version;
 
   public AppLaunchedEvent() {
   }
 
   public AppLaunchedEvent(ApplicationId appId,
       long launchTime, long appSubmitTime, String user,
-      Configuration conf) {
+      Configuration conf, VersionInfo version) {
     this.applicationId = appId;
     this.launchTime = launchTime;
     this.appSubmitTime = appSubmitTime;
     this.user = user;
     this.conf = conf;
+    this.version = version;
   }
 
   @Override
@@ -101,4 +103,8 @@ public class AppLaunchedEvent implements HistoryEvent {
     return conf;
   }
 
+  public VersionInfo getVersion() {
+    return version;
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/tez/blob/6607e944/tez-dag/src/main/java/org/apache/tez/dag/history/utils/DAGUtils.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/utils/DAGUtils.java b/tez-dag/src/main/java/org/apache/tez/dag/history/utils/DAGUtils.java
index 5f259989..0f34811 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/history/utils/DAGUtils.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/history/utils/DAGUtils.java
@@ -30,6 +30,7 @@ import java.util.TreeMap;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.tez.common.ATSConstants;
+import org.apache.tez.common.VersionInfo;
 import org.apache.tez.common.counters.CounterGroup;
 import org.apache.tez.common.counters.TezCounter;
 import org.apache.tez.common.counters.TezCounters;
@@ -371,4 +372,13 @@ public class DAGUtils {
     return atsConf;
   }
 
+  public static Map<String, Object> convertTezVersionToATSMap(VersionInfo versionInfo)
{
+    Map<String, Object> atsInfo = new TreeMap<String, Object>();
+    atsInfo.put(ATSConstants.VERSION, versionInfo.getVersion());
+    atsInfo.put(ATSConstants.BUILD_TIME, versionInfo.getBuildTime());
+    atsInfo.put(ATSConstants.REVISION, versionInfo.getRevision());
+    return atsInfo;
+  }
+
+
 }

http://git-wip-us.apache.org/repos/asf/tez/blob/6607e944/tez-dag/src/test/java/org/apache/tez/dag/history/events/TestHistoryEventsProtoConversion.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/history/events/TestHistoryEventsProtoConversion.java
b/tez-dag/src/test/java/org/apache/tez/dag/history/events/TestHistoryEventsProtoConversion.java
index a7a23db..a0c3231 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/history/events/TestHistoryEventsProtoConversion.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/history/events/TestHistoryEventsProtoConversion.java
@@ -119,7 +119,7 @@ public class TestHistoryEventsProtoConversion {
 
   private void testAppLaunchedEvent() throws Exception {
     AppLaunchedEvent event = new AppLaunchedEvent(ApplicationId.newInstance(0, 1),
-        100, 100, null, new Configuration(false));
+        100, 100, null, new Configuration(false), null);
     try {
       testProtoConversion(event);
       fail("Expected to fail on conversion");

http://git-wip-us.apache.org/repos/asf/tez/blob/6607e944/tez-dag/src/test/java/org/apache/tez/dag/history/logging/impl/TestHistoryEventJsonConversion.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/history/logging/impl/TestHistoryEventJsonConversion.java
b/tez-dag/src/test/java/org/apache/tez/dag/history/logging/impl/TestHistoryEventJsonConversion.java
index c3d51c3..7b30af9 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/history/logging/impl/TestHistoryEventJsonConversion.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/history/logging/impl/TestHistoryEventJsonConversion.java
@@ -105,7 +105,7 @@ public class TestHistoryEventJsonConversion {
       switch (eventType) {
         case APP_LAUNCHED:
           event = new AppLaunchedEvent(applicationId, random.nextInt(), random.nextInt(),
-              user, new Configuration(false));
+              user, new Configuration(false), null);
           break;
         case AM_LAUNCHED:
           event = new AMLaunchedEvent(applicationAttemptId, random.nextInt(), random.nextInt(),

http://git-wip-us.apache.org/repos/asf/tez/blob/6607e944/tez-plugins/tez-yarn-timeline-history/src/main/java/org/apache/tez/dag/history/logging/ats/HistoryEventTimelineConversion.java
----------------------------------------------------------------------
diff --git a/tez-plugins/tez-yarn-timeline-history/src/main/java/org/apache/tez/dag/history/logging/ats/HistoryEventTimelineConversion.java
b/tez-plugins/tez-yarn-timeline-history/src/main/java/org/apache/tez/dag/history/logging/ats/HistoryEventTimelineConversion.java
index 19e2741..3e355e6 100644
--- a/tez-plugins/tez-yarn-timeline-history/src/main/java/org/apache/tez/dag/history/logging/ats/HistoryEventTimelineConversion.java
+++ b/tez-plugins/tez-yarn-timeline-history/src/main/java/org/apache/tez/dag/history/logging/ats/HistoryEventTimelineConversion.java
@@ -59,7 +59,7 @@ public class HistoryEventTimelineConversion {
       throw new UnsupportedOperationException("Invalid Event, does not support history"
           + ", eventType=" + historyEvent.getEventType());
     }
-    TimelineEntity timelineEntity = null;
+    TimelineEntity timelineEntity;
     switch (historyEvent.getEventType()) {
       case APP_LAUNCHED:
         timelineEntity = convertAppLaunchedEvent((AppLaunchedEvent) historyEvent);
@@ -144,6 +144,11 @@ public class HistoryEventTimelineConversion {
 
     atsEntity.setStartTime(event.getLaunchTime());
 
+    if (event.getVersion() != null) {
+      atsEntity.addOtherInfo(ATSConstants.TEZ_VERSION,
+          DAGUtils.convertTezVersionToATSMap(event.getVersion()));
+    }
+
     return atsEntity;
   }
 

http://git-wip-us.apache.org/repos/asf/tez/blob/6607e944/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestHistoryEventTimelineConversion.java
----------------------------------------------------------------------
diff --git a/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestHistoryEventTimelineConversion.java
b/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestHistoryEventTimelineConversion.java
index fe3d780..f32704e 100644
--- a/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestHistoryEventTimelineConversion.java
+++ b/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestHistoryEventTimelineConversion.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
 import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent;
 import org.apache.tez.common.ATSConstants;
+import org.apache.tez.common.VersionInfo;
 import org.apache.tez.common.counters.TezCounters;
 import org.apache.tez.dag.api.EdgeManagerPluginDescriptor;
 import org.apache.tez.dag.api.oldrecords.TaskAttemptState;
@@ -108,7 +109,7 @@ public class TestHistoryEventTimelineConversion {
       switch (eventType) {
         case APP_LAUNCHED:
           event = new AppLaunchedEvent(applicationId, random.nextInt(), random.nextInt(),
-              user, new Configuration(false));
+              user, new Configuration(false), null);
           break;
         case AM_LAUNCHED:
           event = new AMLaunchedEvent(applicationAttemptId, random.nextInt(), random.nextInt(),
@@ -193,6 +194,14 @@ public class TestHistoryEventTimelineConversion {
     }
   }
 
+  static class MockVersionInfo extends VersionInfo {
+
+    MockVersionInfo() {
+      super("component", "1.1.0", "rev1", "20120101", "git.apache.org");
+    }
+
+  }
+
   @Test
   public void testConvertAppLaunchedEvent() {
     long launchTime = random.nextLong();
@@ -201,9 +210,9 @@ public class TestHistoryEventTimelineConversion {
     conf.set("foo", "bar");
     conf.set("applicationId", "1234");
 
-
+    MockVersionInfo mockVersionInfo = new MockVersionInfo();
     AppLaunchedEvent event = new AppLaunchedEvent(applicationId, launchTime,
-        submitTime, user, conf);
+        submitTime, user, conf, mockVersionInfo);
 
     TimelineEntity timelineEntity = HistoryEventTimelineConversion.convertToTimelineEntity(event);
 
@@ -221,13 +230,24 @@ public class TestHistoryEventTimelineConversion {
     Assert.assertEquals(1, timelineEntity.getPrimaryFilters().size());
     Assert.assertTrue(timelineEntity.getPrimaryFilters().get(ATSConstants.USER).contains(user));
 
-    Assert.assertEquals(1, timelineEntity.getOtherInfo().size());
+    Assert.assertEquals(2, timelineEntity.getOtherInfo().size());
     Assert.assertTrue(timelineEntity.getOtherInfo().containsKey(ATSConstants.CONFIG));
+    Assert.assertTrue(timelineEntity.getOtherInfo().containsKey(ATSConstants.TEZ_VERSION));
 
     Map<String, String> config =
         (Map<String, String>)timelineEntity.getOtherInfo().get(ATSConstants.CONFIG);
     Assert.assertEquals(conf.get("foo"), config.get("foo"));
     Assert.assertEquals(conf.get("applicationId"), config.get("applicationId"));
+
+    Map<String, String> versionInfo =
+        (Map<String, String>)timelineEntity.getOtherInfo().get(ATSConstants.TEZ_VERSION);
+    Assert.assertEquals(mockVersionInfo.getVersion(),
+        versionInfo.get(ATSConstants.VERSION));
+    Assert.assertEquals(mockVersionInfo.getRevision(),
+        versionInfo.get(ATSConstants.REVISION));
+    Assert.assertEquals(mockVersionInfo.getBuildTime(),
+        versionInfo.get(ATSConstants.BUILD_TIME));
+
   }
 
   @Test(timeout = 5000)


Mime
View raw message