Return-Path: X-Original-To: apmail-hadoop-yarn-commits-archive@minotaur.apache.org Delivered-To: apmail-hadoop-yarn-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 9B50D10238 for ; Fri, 28 Feb 2014 17:36:57 +0000 (UTC) Received: (qmail 44707 invoked by uid 500); 28 Feb 2014 17:36:36 -0000 Delivered-To: apmail-hadoop-yarn-commits-archive@hadoop.apache.org Received: (qmail 44507 invoked by uid 500); 28 Feb 2014 17:36:29 -0000 Mailing-List: contact yarn-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: yarn-commits@hadoop.apache.org Delivered-To: mailing list yarn-commits@hadoop.apache.org Received: (qmail 44420 invoked by uid 99); 28 Feb 2014 17:36:26 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 28 Feb 2014 17:36:26 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 28 Feb 2014 17:36:25 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 1DD01238883D; Fri, 28 Feb 2014 17:36:05 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1573017 - in /hadoop/common/branches/branch-2/hadoop-yarn-project: ./ hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/ hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/... Date: Fri, 28 Feb 2014 17:36:04 -0000 To: yarn-commits@hadoop.apache.org From: kasha@apache.org X-Mailer: svnmailer-1.0.9 Message-Id: <20140228173605.1DD01238883D@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: kasha Date: Fri Feb 28 17:36:04 2014 New Revision: 1573017 URL: http://svn.apache.org/r1573017 Log: YARN-1528. Allow setting auth for ZK connections. (kasha) Added: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMZKUtils.java - copied unchanged from r1573014, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMZKUtils.java Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/EmbeddedElectorService.java hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStoreZKClientConnections.java Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt?rev=1573017&r1=1573016&r2=1573017&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt (original) +++ hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt Fri Feb 28 17:36:04 2014 @@ -226,6 +226,8 @@ Release 2.4.0 - UNRELEASED YARN-1301. Added the INFO level log of the non-empty blacklist additions and removals inside ApplicationMasterService. (Tsuyoshi Ozawa via zjshen) + YARN-1528. Allow setting auth for ZK connections. (kasha) + OPTIMIZATIONS BUG FIXES Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java?rev=1573017&r1=1573016&r2=1573017&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java (original) +++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java Fri Feb 28 17:36:04 2014 @@ -335,6 +335,8 @@ public class YarnConfiguration extends C public static final String RM_ZK_ACL = RM_ZK_PREFIX + "acl"; public static final String DEFAULT_RM_ZK_ACL = "world:anyone:rwcda"; + public static final String RM_ZK_AUTH = RM_ZK_PREFIX + "auth"; + public static final String ZK_STATE_STORE_PREFIX = RM_PREFIX + "zk-state-store."; Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/EmbeddedElectorService.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/EmbeddedElectorService.java?rev=1573017&r1=1573016&r2=1573017&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/EmbeddedElectorService.java (original) +++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/EmbeddedElectorService.java Fri Feb 28 17:36:04 2014 @@ -31,14 +31,12 @@ import org.apache.hadoop.util.StringUtil import org.apache.hadoop.util.ZKUtil; import org.apache.hadoop.yarn.conf.HAUtil; import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos; import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.data.ACL; import java.io.IOException; -import java.util.Collections; import java.util.List; @InterfaceAudience.Private @@ -88,18 +86,8 @@ public class EmbeddedElectorService exte long zkSessionTimeout = conf.getLong(YarnConfiguration.RM_ZK_TIMEOUT_MS, YarnConfiguration.DEFAULT_RM_ZK_TIMEOUT_MS); - String zkAclConf = conf.get(YarnConfiguration.RM_ZK_ACL, - YarnConfiguration.DEFAULT_RM_ZK_ACL); - List zkAcls; - try { - zkAcls = ZKUtil.parseACLs(ZKUtil.resolveConfIndirection(zkAclConf)); - } catch (ZKUtil.BadAclFormatException bafe) { - throw new YarnRuntimeException( - YarnConfiguration.RM_ZK_ACL + "has ill-formatted ACLs"); - } - - // TODO (YARN-1528): ZKAuthInfo to be set for rm-store and elector - List zkAuths = Collections.emptyList(); + List zkAcls = RMZKUtils.getZKAcls(conf); + List zkAuths = RMZKUtils.getZKAuths(conf); elector = new ActiveStandbyElector(zkQuorum, (int) zkSessionTimeout, electionZNode, zkAcls, zkAuths, this); Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java?rev=1573017&r1=1573016&r2=1573017&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java (original) +++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java Fri Feb 28 17:36:04 2014 @@ -48,6 +48,7 @@ import org.apache.hadoop.yarn.proto.Yarn import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.ApplicationStateDataProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.RMStateVersionProto; import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier; +import org.apache.hadoop.yarn.server.resourcemanager.RMZKUtils; import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.RMStateVersion; import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.ApplicationAttemptStateDataPBImpl; import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.ApplicationStateDataPBImpl; @@ -91,6 +92,7 @@ public class ZKRMStateStore extends RMSt private int zkSessionTimeout; private long zkRetryInterval; private List zkAcl; + private List zkAuths; /** * @@ -200,18 +202,9 @@ public class ZKRMStateStore extends RMSt zkRetryInterval = conf.getLong(YarnConfiguration.RM_ZK_RETRY_INTERVAL_MS, YarnConfiguration.DEFAULT_RM_ZK_RETRY_INTERVAL_MS); - // Parse authentication from configuration. - String zkAclConf = - conf.get(YarnConfiguration.RM_ZK_ACL, - YarnConfiguration.DEFAULT_RM_ZK_ACL); - zkAclConf = ZKUtil.resolveConfIndirection(zkAclConf); - try { - zkAcl = ZKUtil.parseACLs(zkAclConf); - } catch (ZKUtil.BadAclFormatException bafe) { - LOG.error("Invalid format for " + YarnConfiguration.RM_ZK_ACL); - throw bafe; - } + zkAcl = RMZKUtils.getZKAcls(conf); + zkAuths = RMZKUtils.getZKAuths(conf); zkRootNodePath = getNodePath(znodeWorkingPath, ROOT_ZNODE_NAME); rmAppRoot = getNodePath(zkRootNodePath, RM_APP_ROOT); @@ -952,6 +945,9 @@ public class ZKRMStateStore extends RMSt retries++) { try { zkClient = getNewZooKeeper(); + for (ZKUtil.ZKAuthInfo zkAuth : zkAuths) { + zkClient.addAuthInfo(zkAuth.getScheme(), zkAuth.getAuth()); + } if (useDefaultFencingScheme) { zkClient.addAuthInfo(zkRootNodeAuthScheme, (zkRootNodeUsername + ":" + zkRootNodePassword).getBytes()); Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStoreZKClientConnections.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStoreZKClientConnections.java?rev=1573017&r1=1573016&r2=1573017&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStoreZKClientConnections.java (original) +++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStoreZKClientConnections.java Fri Feb 28 17:36:04 2014 @@ -32,10 +32,12 @@ import org.apache.zookeeper.WatchedEvent import org.apache.zookeeper.Watcher; import org.apache.zookeeper.ZooDefs; import org.apache.zookeeper.ZooKeeper; +import org.apache.zookeeper.server.auth.DigestAuthenticationProvider; import org.junit.Assert; import org.junit.Test; import java.io.IOException; +import java.security.NoSuchAlgorithmException; import java.util.concurrent.CyclicBarrier; import java.util.concurrent.atomic.AtomicBoolean; @@ -49,6 +51,20 @@ public class TestZKRMStateStoreZKClientC private Log LOG = LogFactory.getLog(TestZKRMStateStoreZKClientConnections.class); + private static final String DIGEST_USER_PASS="test-user:test-password"; + private static final String TEST_AUTH_GOOD = "digest:" + DIGEST_USER_PASS; + private static final String DIGEST_USER_HASH; + static { + try { + DIGEST_USER_HASH = DigestAuthenticationProvider.generateDigest( + DIGEST_USER_PASS); + } catch (NoSuchAlgorithmException e) { + throw new RuntimeException(e); + } + } + private static final String TEST_ACL = "digest:" + DIGEST_USER_HASH + ":rwcda"; + + class TestZKClient { ZKRMStateStore store; @@ -252,4 +268,16 @@ public class TestZKRMStateStoreZKClientC fail(error); } } + + @Test + public void testZKAuths() throws Exception { + TestZKClient zkClientTester = new TestZKClient(); + YarnConfiguration conf = new YarnConfiguration(); + conf.setInt(YarnConfiguration.RM_ZK_NUM_RETRIES, 1); + conf.setInt(YarnConfiguration.RM_ZK_TIMEOUT_MS, 100); + conf.set(YarnConfiguration.RM_ZK_ACL, TEST_ACL); + conf.set(YarnConfiguration.RM_ZK_AUTH, TEST_AUTH_GOOD); + + zkClientTester.getRMStateStore(conf); + } }