Return-Path: X-Original-To: apmail-tez-commits-archive@minotaur.apache.org Delivered-To: apmail-tez-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 096E6187F6 for ; Fri, 18 Mar 2016 19:45:58 +0000 (UTC) Received: (qmail 8537 invoked by uid 500); 18 Mar 2016 19:45:58 -0000 Delivered-To: apmail-tez-commits-archive@tez.apache.org Received: (qmail 8516 invoked by uid 500); 18 Mar 2016 19:45:58 -0000 Mailing-List: contact commits-help@tez.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@tez.apache.org Delivered-To: mailing list commits@tez.apache.org Received: (qmail 8506 invoked by uid 99); 18 Mar 2016 19:45:57 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 18 Mar 2016 19:45:57 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id C9AF2DFA40; Fri, 18 Mar 2016 19:45:57 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: jeagles@apache.org To: commits@tez.apache.org Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: tez git commit: TEZ-3175. Add tez client submit host (jeagles) Date: Fri, 18 Mar 2016 19:45:57 +0000 (UTC) Repository: tez Updated Branches: refs/heads/master c9c7d8099 -> dacd0191b TEZ-3175. Add tez client submit host (jeagles) Project: http://git-wip-us.apache.org/repos/asf/tez/repo Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/dacd0191 Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/dacd0191 Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/dacd0191 Branch: refs/heads/master Commit: dacd0191b684208d71ea457ca849f2d01212bb7e Parents: c9c7d80 Author: Jonathan Eagles Authored: Fri Mar 18 14:45:40 2016 -0500 Committer: Jonathan Eagles Committed: Fri Mar 18 14:45:40 2016 -0500 ---------------------------------------------------------------------- CHANGES.txt | 2 ++ .../main/java/org/apache/tez/client/TezClient.java | 12 ++++++++++++ .../org/apache/tez/dag/api/TezConfiguration.java | 14 ++++++++++++++ .../java/org/apache/tez/client/TestTezClient.java | 16 ++++++++++++++++ 4 files changed, 44 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tez/blob/dacd0191/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index 30e4fc6..f7dd9d9 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -7,6 +7,7 @@ INCOMPATIBLE CHANGES TEZ-3029. Add an onError method to service plugin contexts. ALL CHANGES: + TEZ-3175. Add tez client submit host TEZ-3166. Fix a few cases where counters aren't fully updated and sent for failed tasks. TEZ-2958. Recovered TA, whose commit cannot be recovered, should move to killed state TEZ-2936. Create ATS implementation that enables support for YARN-4265 (ATS v1.5) @@ -406,6 +407,7 @@ INCOMPATIBLE CHANGES TEZ-2949. Allow duplicate dag names within session for Tez. ALL CHANGES: + TEZ-3175. Add tez client submit host TEZ-3166. Fix a few cases where counters aren't fully updated and sent for failed tasks. TEZ-2958. Recovered TA, whose commit cannot be recovered, should move to killed state TEZ-3105. TezMxBeanResourceCalculator does not work on IBM JDK 7 or 8 causing Tez failures. http://git-wip-us.apache.org/repos/asf/tez/blob/dacd0191/tez-api/src/main/java/org/apache/tez/client/TezClient.java ---------------------------------------------------------------------- diff --git a/tez-api/src/main/java/org/apache/tez/client/TezClient.java b/tez-api/src/main/java/org/apache/tez/client/TezClient.java index bded5a2..43cdfb8 100644 --- a/tez-api/src/main/java/org/apache/tez/client/TezClient.java +++ b/tez-api/src/main/java/org/apache/tez/client/TezClient.java @@ -19,6 +19,8 @@ package org.apache.tez.client; import java.io.IOException; +import java.net.InetAddress; +import java.net.UnknownHostException; import java.text.NumberFormat; import java.util.HashMap; import java.util.Map; @@ -182,6 +184,16 @@ public class TezClient { this.isSession = isSession; // Set in conf for local mode AM to figure out whether in session mode or not tezConf.setBoolean(TezConfiguration.TEZ_AM_SESSION_MODE, isSession); + try { + InetAddress ip = InetAddress.getLocalHost(); + if (ip != null) { + tezConf.set(TezConfiguration.TEZ_SUBMIT_HOST, ip.getCanonicalHostName()); + tezConf.set(TezConfiguration.TEZ_SUBMIT_HOST_ADDRESS, ip.getHostAddress()); + } + } catch (UnknownHostException e) { + LOG.warn("The host name of the client the tez application was submitted from was unable to be retrieved", e); + } + this.amConfig = new AMConfiguration(tezConf, localResources, credentials); this.apiVersionInfo = new TezApiVersionInfo(); this.servicePluginsDescriptor = servicePluginsDescriptor; http://git-wip-us.apache.org/repos/asf/tez/blob/dacd0191/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java ---------------------------------------------------------------------- diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java index 4fc8322..924c7ff 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java @@ -1106,6 +1106,20 @@ public class TezConfiguration extends Configuration { @ConfigurationScope(Scope.AM) public static final String TEZ_AM_APPLICATION_PRIORITY = TEZ_PREFIX + "am.application.priority"; + /** + * String value. Set automatically by the client. The host name of the client the Tez application was submitted from. + */ + @Private + @ConfigurationScope(Scope.AM) + public static final String TEZ_SUBMIT_HOST = TEZ_PREFIX + "submit.host"; + + /** + * String value. Set automatically by the client. The host address of the client the Tez application was submitted from. + */ + @Private + @ConfigurationScope(Scope.AM) + public static final String TEZ_SUBMIT_HOST_ADDRESS = TEZ_PREFIX + "submit.host.address"; + @Unstable /** * Boolean value. Generate debug artifacts like DAG plan in text format. http://git-wip-us.apache.org/repos/asf/tez/blob/dacd0191/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java ---------------------------------------------------------------------- diff --git a/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java b/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java index a2e4956..ae822f3 100644 --- a/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java +++ b/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java @@ -19,6 +19,8 @@ package org.apache.tez.client; import java.io.IOException; +import java.net.InetAddress; +import java.net.UnknownHostException; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Collections; @@ -635,4 +637,18 @@ public class TestTezClient { verify(client.mockYarnClient, atLeast(2)).getApplicationReport(client.mockAppId); Assert.assertTrue("Stop ended before timeout", end - start > HARD_KILL_TIMEOUT); } + + @Test(timeout = 5000) + public void testSubmitHostPopulated() throws YarnException, IOException, ServiceException, TezException { + + TezConfiguration conf = new TezConfiguration(); + configureAndCreateTezClient(conf); + InetAddress ip = InetAddress.getLocalHost(); + if (ip != null) { + Assert.assertEquals(ip.getCanonicalHostName(), conf.get(TezConfiguration.TEZ_SUBMIT_HOST)); + Assert.assertEquals(ip.getHostAddress(), conf.get(TezConfiguration.TEZ_SUBMIT_HOST_ADDRESS)); + } else { + Assert.fail("Failed to retrieve local host information"); + } + } }