Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 54DAA200C45 for ; Tue, 28 Mar 2017 21:47:45 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 538E7160BC4; Tue, 28 Mar 2017 19:47:45 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id E373D160BA2 for ; Tue, 28 Mar 2017 21:47:43 +0200 (CEST) Received: (qmail 70554 invoked by uid 500); 28 Mar 2017 19:47:43 -0000 Mailing-List: contact commits-help@tez.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@tez.apache.org Delivered-To: mailing list commits@tez.apache.org Received: (qmail 70236 invoked by uid 99); 28 Mar 2017 19:47:42 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 28 Mar 2017 19:47:42 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id AA4FAE69B9; Tue, 28 Mar 2017 19:47:42 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: zhiyuany@apache.org To: commits@tez.apache.org Date: Tue, 28 Mar 2017 19:47:49 -0000 Message-Id: <9ca0931677a9458aa177b1adec2129b8@git.apache.org> In-Reply-To: <3c41745652fc4012b06b29e1b1cbf344@git.apache.org> References: <3c41745652fc4012b06b29e1b1cbf344@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [08/50] [abbrv] tez git commit: TEZ-3550. Provide access to sessionId/dagId via DagClient. (sseth) archived-at: Tue, 28 Mar 2017 19:47:45 -0000 TEZ-3550. Provide access to sessionId/dagId via DagClient. (sseth) Project: http://git-wip-us.apache.org/repos/asf/tez/repo Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/cd44c906 Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/cd44c906 Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/cd44c906 Branch: refs/heads/TEZ-1190 Commit: cd44c906f762c24a13ddabe691e55c879c223eac Parents: e1b0b28 Author: Siddharth Seth Authored: Tue Feb 14 10:46:18 2017 -0800 Committer: Siddharth Seth Committed: Tue Feb 14 10:46:18 2017 -0800 ---------------------------------------------------------------------- CHANGES.txt | 2 ++ .../java/org/apache/tez/dag/api/client/DAGClient.java | 13 +++++++++++++ .../org/apache/tez/dag/api/client/DAGClientImpl.java | 10 ++++++++++ .../apache/tez/dag/api/client/DAGClientInternal.java | 14 +++++++++++++- .../tez/dag/api/client/DAGClientTimelineImpl.java | 11 ++++++++++- .../tez/dag/api/client/rpc/DAGClientRPCImpl.java | 10 +++++++++- .../java/org/apache/tez/client/TestTezClient.java | 5 ++++- .../apache/tez/dag/api/client/rpc/TestDAGClient.java | 2 ++ .../org/apache/tez/dag/api/client/MRDAGClient.java | 10 ++++++++++ 9 files changed, 73 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tez/blob/cd44c906/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index 6991f05..337b394 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -7,6 +7,7 @@ INCOMPATIBLE CHANGES ALL CHANGES: + TEZ-3550. Provide access to sessionId/dagId via DagClient. TEZ-3267. Publish queue name to ATS as part of dag summary. TEZ-3609. Improve ATSv15 performance for DAG entities read calls. TEZ-3244. Allow overlap of input and output memory when they are not concurrent @@ -201,6 +202,7 @@ INCOMPATIBLE CHANGES ALL CHANGES: + TEZ-3550. Provide access to sessionId/dagId via DagClient. TEZ-3609. Improve ATSv15 performance for DAG entities read calls. TEZ-3244. Allow overlap of input and output memory when they are not concurrent TEZ-3601. Add another HistoryLogLevel to suppress TaskAttempts at specific levels http://git-wip-us.apache.org/repos/asf/tez/blob/cd44c906/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClient.java ---------------------------------------------------------------------- diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClient.java b/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClient.java index 9b11b96..c70da75 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClient.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClient.java @@ -84,6 +84,19 @@ public abstract class DAGClient implements Closeable { throws IOException, TezException; /** + * Get the dag identifier for the currently executing dag. This is a string + * which represents this dag + * @return the dag identifier + */ + public abstract String getDagIdentifierString(); + + /** + * Get the session identifier for the session in which this dag is running + * @return the session identifier + */ + public abstract String getSessionIdentifierString(); + + /** * Kill a running DAG * */ http://git-wip-us.apache.org/repos/asf/tez/blob/cd44c906/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClientImpl.java ---------------------------------------------------------------------- diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClientImpl.java b/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClientImpl.java index 4820b6e..1cf0bfc 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClientImpl.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClientImpl.java @@ -319,6 +319,16 @@ public class DAGClientImpl extends DAGClient { } @Override + public String getDagIdentifierString() { + return realClient.getDagIdentifierString(); + } + + @Override + public String getSessionIdentifierString() { + return realClient.getSessionIdentifierString(); + } + + @Override public void tryKillDAG() throws IOException, TezException { if (!dagCompleted) { realClient.tryKillDAG(); http://git-wip-us.apache.org/repos/asf/tez/blob/cd44c906/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClientInternal.java ---------------------------------------------------------------------- diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClientInternal.java b/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClientInternal.java index bb236a3..a3c898a 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClientInternal.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClientInternal.java @@ -25,7 +25,6 @@ import java.util.Set; import javax.annotation.Nullable; import org.apache.hadoop.classification.InterfaceAudience.Private; -import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException; import org.apache.tez.dag.api.TezException; @@ -85,6 +84,19 @@ public abstract class DAGClientInternal implements Closeable { throws IOException, TezException, ApplicationNotFoundException; /** + * Get the dag identifier for the currently executing dag. This is a string + * which represents this dag + * @return the dag identifier + */ + public abstract String getDagIdentifierString(); + + /** + * Get the session identifier for the session in which this dag is running + * @return the session identifier + */ + public abstract String getSessionIdentifierString(); + + /** * Kill a running DAG * */ http://git-wip-us.apache.org/repos/asf/tez/blob/cd44c906/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClientTimelineImpl.java ---------------------------------------------------------------------- diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClientTimelineImpl.java b/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClientTimelineImpl.java index ffd91b7..d34dbf0 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClientTimelineImpl.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClientTimelineImpl.java @@ -38,7 +38,6 @@ import com.sun.jersey.api.client.ClientResponse; import com.sun.jersey.api.client.UniformInterfaceException; import com.sun.jersey.api.client.WebResource; -import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; @@ -182,6 +181,16 @@ public class DAGClientTimelineImpl extends DAGClientInternal { } @Override + public String getDagIdentifierString() { + return dagId; + } + + @Override + public String getSessionIdentifierString() { + return appId.toString(); + } + + @Override public void tryKillDAG() throws IOException, TezException { throw new TezException("tryKillDAG is unsupported for DAGClientTimelineImpl"); } http://git-wip-us.apache.org/repos/asf/tez/blob/cd44c906/tez-api/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientRPCImpl.java ---------------------------------------------------------------------- diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientRPCImpl.java b/tez-api/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientRPCImpl.java index 9eb9807..02935df 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientRPCImpl.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientRPCImpl.java @@ -41,7 +41,6 @@ import org.apache.tez.dag.api.DAGNotRunningException; import org.apache.tez.dag.api.DagTypeConverters; import org.apache.tez.dag.api.TezConfiguration; import org.apache.tez.dag.api.TezException; -import org.apache.tez.dag.api.client.DAGClient; import org.apache.tez.dag.api.client.DAGStatus; import org.apache.tez.dag.api.client.DagStatusSource; import org.apache.tez.dag.api.client.StatusGetOpts; @@ -130,6 +129,15 @@ public class DAGClientRPCImpl extends DAGClientInternal { return null; } + @Override + public String getDagIdentifierString() { + return dagId.toString(); + } + + @Override + public String getSessionIdentifierString() { + return appId.toString(); + } @Override http://git-wip-us.apache.org/repos/asf/tez/blob/cd44c906/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java ---------------------------------------------------------------------- diff --git a/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java b/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java index c9415a4..89310df 100644 --- a/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java +++ b/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java @@ -277,7 +277,8 @@ public class TestTezClient { DAGClient dagClient = client.submitDAG(dag); assertTrue(dagClient.getExecutionContext().contains(client.mockAppId.toString())); - + assertEquals(dagClient.getSessionIdentifierString(), client.mockAppId.toString()); + if (isSession) { verify(client.mockYarnClient, times(1)).submitApplication(captor.capture()); verify(client.sessionAmProxy, times(1)).submitDAG((RpcController)any(), (SubmitDAGRequestProto) any()); @@ -316,6 +317,7 @@ public class TestTezClient { // same app master verify(client.mockYarnClient, times(1)).submitApplication(captor.capture()); assertTrue(dagClient.getExecutionContext().contains(client.mockAppId.toString())); + assertEquals(dagClient.getSessionIdentifierString(), client.mockAppId.toString()); // additional resource is sent ArgumentCaptor captor1 = ArgumentCaptor.forClass(SubmitDAGRequestProto.class); verify(client.sessionAmProxy, times(2)).submitDAG((RpcController)any(), captor1.capture()); @@ -325,6 +327,7 @@ public class TestTezClient { } else { // new app master assertTrue(dagClient.getExecutionContext().contains(appId2.toString())); + assertEquals(dagClient.getSessionIdentifierString(), appId2.toString()); verify(client.mockYarnClient, times(2)).submitApplication(captor.capture()); // additional resource is added ApplicationSubmissionContext context = captor.getValue(); http://git-wip-us.apache.org/repos/asf/tez/blob/cd44c906/tez-api/src/test/java/org/apache/tez/dag/api/client/rpc/TestDAGClient.java ---------------------------------------------------------------------- diff --git a/tez-api/src/test/java/org/apache/tez/dag/api/client/rpc/TestDAGClient.java b/tez-api/src/test/java/org/apache/tez/dag/api/client/rpc/TestDAGClient.java index e979f8b..70ee1d4 100644 --- a/tez-api/src/test/java/org/apache/tez/dag/api/client/rpc/TestDAGClient.java +++ b/tez-api/src/test/java/org/apache/tez/dag/api/client/rpc/TestDAGClient.java @@ -213,6 +213,8 @@ public class TestDAGClient { @Test(timeout = 5000) public void testApp() throws IOException, TezException, ServiceException{ assertTrue(dagClient.getExecutionContext().contains(mockAppId.toString())); + assertEquals(mockAppId.toString(), dagClient.getSessionIdentifierString()); + assertEquals(dagIdStr, dagClient.getDagIdentifierString()); DAGClientRPCImpl realClient = (DAGClientRPCImpl)((DAGClientImpl)dagClient).getRealClient(); assertEquals(mockAppReport, realClient.getApplicationReportInternal()); } http://git-wip-us.apache.org/repos/asf/tez/blob/cd44c906/tez-mapreduce/src/main/java/org/apache/tez/dag/api/client/MRDAGClient.java ---------------------------------------------------------------------- diff --git a/tez-mapreduce/src/main/java/org/apache/tez/dag/api/client/MRDAGClient.java b/tez-mapreduce/src/main/java/org/apache/tez/dag/api/client/MRDAGClient.java index d743feb..42b52e0 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/dag/api/client/MRDAGClient.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/dag/api/client/MRDAGClient.java @@ -66,6 +66,16 @@ public class MRDAGClient extends DAGClient { } @Override + public String getDagIdentifierString() { + return realClient.getDagIdentifierString(); + } + + @Override + public String getSessionIdentifierString() { + return realClient.getSessionIdentifierString(); + } + + @Override public void tryKillDAG() throws IOException, TezException { realClient.tryKillDAG(); }