From common-commits-return-84514-archive-asf-public=cust-asf.ponee.io@hadoop.apache.org Sun Jun 17 09:10:21 2018 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx-eu-01.ponee.io (Postfix) with SMTP id 1B24118062A for ; Sun, 17 Jun 2018 09:10:20 +0200 (CEST) Received: (qmail 63312 invoked by uid 500); 17 Jun 2018 07:10:20 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 63303 invoked by uid 99); 17 Jun 2018 07:10:20 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Sun, 17 Jun 2018 07:10:20 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id CCB52E0C64; Sun, 17 Jun 2018 07:10:19 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: surendralilhore@apache.org To: common-commits@hadoop.apache.org Message-Id: <7d07d897898d4d30806b029d4ecd8391@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: hadoop git commit: HADOOP-15523. Shell command timeout given is in seconds whereas it is taken as millisec while scheduling. Contributed by Bilwa S T. Date: Sun, 17 Jun 2018 07:10:19 +0000 (UTC) Repository: hadoop Updated Branches: refs/heads/branch-3.1 88e8a3e5b -> 40652e77e HADOOP-15523. Shell command timeout given is in seconds whereas it is taken as millisec while scheduling. Contributed by Bilwa S T. (cherry picked from commit 3905fdb793e6370243d05d0c3036ca69898fe3fb) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/40652e77 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/40652e77 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/40652e77 Branch: refs/heads/branch-3.1 Commit: 40652e77e057cb640e14a4238299516ca5b4169f Parents: 88e8a3e Author: Surendra Singh Lilhore Authored: Sun Jun 17 12:12:01 2018 +0530 Committer: Surendra Singh Lilhore Committed: Sun Jun 17 12:14:20 2018 +0530 ---------------------------------------------------------------------- .../fs/CommonConfigurationKeysPublic.java | 4 +- .../security/ShellBasedUnixGroupsMapping.java | 10 ++--- .../main/java/org/apache/hadoop/util/Shell.java | 2 +- .../TestShellBasedUnixGroupsMapping.java | 39 ++++++++++++++++++-- 4 files changed, 43 insertions(+), 12 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/40652e77/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java index b0413a9..132c9bf 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java @@ -542,7 +542,7 @@ public class CommonConfigurationKeysPublic { * * core-default.xml */ - public static final String HADOOP_SECURITY_GROUP_SHELL_COMMAND_TIMEOUT_SECS = + public static final String HADOOP_SECURITY_GROUP_SHELL_COMMAND_TIMEOUT_KEY = "hadoop.security.groups.shell.command.timeout"; /** * @see @@ -550,7 +550,7 @@ public class CommonConfigurationKeysPublic { * core-default.xml */ public static final long - HADOOP_SECURITY_GROUP_SHELL_COMMAND_TIMEOUT_SECS_DEFAULT = + HADOOP_SECURITY_GROUP_SHELL_COMMAND_TIMEOUT_DEFAULT = 0L; /** * @see http://git-wip-us.apache.org/repos/asf/hadoop/blob/40652e77/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedUnixGroupsMapping.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedUnixGroupsMapping.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedUnixGroupsMapping.java index 94698d8..976ddba 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedUnixGroupsMapping.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedUnixGroupsMapping.java @@ -18,7 +18,6 @@ package org.apache.hadoop.security; import java.io.IOException; -import java.util.Arrays; import java.util.LinkedList; import java.util.List; import java.util.StringTokenizer; @@ -52,7 +51,8 @@ public class ShellBasedUnixGroupsMapping extends Configured protected static final Logger LOG = LoggerFactory.getLogger(ShellBasedUnixGroupsMapping.class); - private long timeout = 0L; + private long timeout = CommonConfigurationKeys. + HADOOP_SECURITY_GROUP_SHELL_COMMAND_TIMEOUT_DEFAULT; private static final List EMPTY_GROUPS = new LinkedList<>(); @Override @@ -61,10 +61,10 @@ public class ShellBasedUnixGroupsMapping extends Configured if (conf != null) { timeout = conf.getTimeDuration( CommonConfigurationKeys. - HADOOP_SECURITY_GROUP_SHELL_COMMAND_TIMEOUT_SECS, + HADOOP_SECURITY_GROUP_SHELL_COMMAND_TIMEOUT_KEY, CommonConfigurationKeys. - HADOOP_SECURITY_GROUP_SHELL_COMMAND_TIMEOUT_SECS_DEFAULT, - TimeUnit.SECONDS); + HADOOP_SECURITY_GROUP_SHELL_COMMAND_TIMEOUT_DEFAULT, + TimeUnit.MILLISECONDS); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/40652e77/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java index 04b4b4f..7955daa 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java @@ -1205,7 +1205,7 @@ public abstract class Shell { /** * Returns the timeout value set for the executor's sub-commands. - * @return The timeout value in seconds + * @return The timeout value in milliseconds */ @VisibleForTesting public long getTimeoutInterval() { http://git-wip-us.apache.org/repos/asf/hadoop/blob/40652e77/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestShellBasedUnixGroupsMapping.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestShellBasedUnixGroupsMapping.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestShellBasedUnixGroupsMapping.java index d3c9538..8c1339d 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestShellBasedUnixGroupsMapping.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestShellBasedUnixGroupsMapping.java @@ -173,6 +173,37 @@ public class TestShellBasedUnixGroupsMapping { assertTrue(groups.contains("zzz")); } + public long getTimeoutInterval(String timeout) { + Configuration conf = new Configuration(); + String userName = "foobarnonexistinguser"; + conf.set( + CommonConfigurationKeys.HADOOP_SECURITY_GROUP_SHELL_COMMAND_TIMEOUT_KEY, + timeout); + TestDelayedGroupCommand mapping = ReflectionUtils + .newInstance(TestDelayedGroupCommand.class, conf); + ShellCommandExecutor executor = mapping.createGroupExecutor(userName); + return executor.getTimeoutInterval(); + } + + @Test + public void testShellTimeOutConf() { + + // Test a 1 second max-runtime timeout + assertEquals( + "Expected the group names executor to carry the configured timeout", + 1000L, getTimeoutInterval("1s")); + + // Test a 1 minute max-runtime timeout + assertEquals( + "Expected the group names executor to carry the configured timeout", + 60000L, getTimeoutInterval("1m")); + + // Test a 1 millisecond max-runtime timeout + assertEquals( + "Expected the group names executor to carry the configured timeout", + 1L, getTimeoutInterval("1")); + } + private class TestGroupResolvable extends ShellBasedUnixGroupsMapping { /** @@ -222,7 +253,7 @@ public class TestShellBasedUnixGroupsMapping { private static class TestDelayedGroupCommand extends ShellBasedUnixGroupsMapping { - private Long timeoutSecs = 2L; + private Long timeoutSecs = 1L; TestDelayedGroupCommand() { super(); @@ -249,12 +280,12 @@ public class TestShellBasedUnixGroupsMapping { String userName = "foobarnonexistinguser"; String commandTimeoutMessage = "ran longer than the configured timeout limit"; - long testTimeout = 1L; + long testTimeout = 500L; // Test a 1 second max-runtime timeout conf.setLong( CommonConfigurationKeys. - HADOOP_SECURITY_GROUP_SHELL_COMMAND_TIMEOUT_SECS, + HADOOP_SECURITY_GROUP_SHELL_COMMAND_TIMEOUT_KEY, testTimeout); TestDelayedGroupCommand mapping = @@ -306,7 +337,7 @@ public class TestShellBasedUnixGroupsMapping { conf = new Configuration(); long defaultTimeout = CommonConfigurationKeys. - HADOOP_SECURITY_GROUP_SHELL_COMMAND_TIMEOUT_SECS_DEFAULT; + HADOOP_SECURITY_GROUP_SHELL_COMMAND_TIMEOUT_DEFAULT; mapping = ReflectionUtils.newInstance(TestDelayedGroupCommand.class, conf); --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org