Return-Path: X-Original-To: apmail-hadoop-yarn-commits-archive@minotaur.apache.org Delivered-To: apmail-hadoop-yarn-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id BFD20101C3 for ; Sat, 31 May 2014 00:01:55 +0000 (UTC) Received: (qmail 16738 invoked by uid 500); 31 May 2014 00:01:55 -0000 Delivered-To: apmail-hadoop-yarn-commits-archive@hadoop.apache.org Received: (qmail 16692 invoked by uid 500); 31 May 2014 00:01:55 -0000 Mailing-List: contact yarn-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: yarn-commits@hadoop.apache.org Delivered-To: mailing list yarn-commits@hadoop.apache.org Received: (qmail 16683 invoked by uid 99); 31 May 2014 00:01:55 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 31 May 2014 00:01:55 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 31 May 2014 00:01:51 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 3DD362388A9B; Sat, 31 May 2014 00:01:31 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1598785 [1/2] - in /hadoop/common/branches/fs-encryption/hadoop-yarn-project: ./ hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/ hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/ hadoop... Date: Sat, 31 May 2014 00:01:26 -0000 To: yarn-commits@hadoop.apache.org From: clamb@apache.org X-Mailer: svnmailer-1.0.9 Message-Id: <20140531000131.3DD362388A9B@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: clamb Date: Sat May 31 00:01:20 2014 New Revision: 1598785 URL: http://svn.apache.org/r1598785 Log: merge from trunk r1598783 Added: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/event/ResourceRecoveredEvent.java - copied unchanged from r1598783, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/event/ResourceRecoveredEvent.java hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/ - copied from r1598783, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java - copied unchanged from r1598783, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMNullStateStoreService.java - copied unchanged from r1598783, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMNullStateStoreService.java hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMStateStoreService.java - copied unchanged from r1598783, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMStateStoreService.java hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/proto/yarn_server_nodemanager_recovery.proto - copied unchanged from r1598783, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/proto/yarn_server_nodemanager_recovery.proto hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/ - copied from r1598783, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMMemoryStateStoreService.java - copied unchanged from r1598783, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMMemoryStateStoreService.java hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/TestNMLeveldbStateStoreService.java - copied unchanged from r1598783, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/TestNMLeveldbStateStoreService.java Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/CHANGES.txt hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/HtmlPage.java hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/TestSubViews.java hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/view/TestHtmlPage.java hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalCacheDirectoryManager.java hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTracker.java hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalizedResource.java hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/event/ResourceEventType.java hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalCacheDirectoryManager.java hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResourcesTrackerImpl.java hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceRetention.java hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestContainerLogsPage.java hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStoreZKClientConnections.java Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/CHANGES.txt?rev=1598785&r1=1598784&r2=1598785&view=diff ============================================================================== --- hadoop/common/branches/fs-encryption/hadoop-yarn-project/CHANGES.txt (original) +++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/CHANGES.txt Sat May 31 00:01:20 2014 @@ -29,6 +29,9 @@ Release 2.5.0 - UNRELEASED YARN-1362. Distinguish between nodemanager shutdown for decommission vs shutdown for restart. (Jason Lowe via junping_du) + YARN-1338. Recover localized resource cache state upon nodemanager restart + (Jason Lowe via junping_du) + IMPROVEMENTS YARN-1479. Invalid NaN values in Hadoop REST API JSON response (Chen He via @@ -117,6 +120,12 @@ Release 2.5.0 - UNRELEASED YARN-596. Use scheduling policies throughout the queue hierarchy to decide which containers to preempt (Wei Yan via Sandy Ryza) + YARN-2054. Better defaults for YARN ZK configs for retries and retry-inteval + when HA is enabled. (kasha) + + YARN-2010. Document yarn.resourcemanager.zk-auth and its scope. + (Robert Kanter via kasha) + OPTIMIZATIONS BUG FIXES @@ -162,6 +171,9 @@ Release 2.5.0 - UNRELEASED that both Timeline Server and client can access them. (Zhijie Shen via vinodkv) + YARN-1868. YARN status web ui does not show correctly in IE 11. + (Chuan Liu via cnauroth) + Release 2.4.1 - UNRELEASED INCOMPATIBLE CHANGES Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java?rev=1598785&r1=1598784&r2=1598785&view=diff ============================================================================== --- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java (original) +++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java Sat May 31 00:01:20 2014 @@ -324,11 +324,11 @@ public class YarnConfiguration extends C public static final String RM_ZK_ADDRESS = RM_ZK_PREFIX + "address"; public static final String RM_ZK_NUM_RETRIES = RM_ZK_PREFIX + "num-retries"; - public static final int DEFAULT_ZK_RM_NUM_RETRIES = 500; + public static final int DEFAULT_ZK_RM_NUM_RETRIES = 1000; public static final String RM_ZK_RETRY_INTERVAL_MS = RM_ZK_PREFIX + "retry-interval-ms"; - public static final long DEFAULT_RM_ZK_RETRY_INTERVAL_MS = 2000; + public static final long DEFAULT_RM_ZK_RETRY_INTERVAL_MS = 1000; public static final String RM_ZK_TIMEOUT_MS = RM_ZK_PREFIX + "timeout-ms"; public static final int DEFAULT_RM_ZK_TIMEOUT_MS = 10000; Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/HtmlPage.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/HtmlPage.java?rev=1598785&r1=1598784&r2=1598785&view=diff ============================================================================== --- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/HtmlPage.java (original) +++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/HtmlPage.java Sat May 31 00:01:20 2014 @@ -79,7 +79,8 @@ public abstract class HtmlPage extends T @Override public void render() { puts(DOCTYPE); - render(page().html().meta_http("Content-type", MimeType.HTML)); + render(page().html().meta_http("X-UA-Compatible", "IE=8") + .meta_http("Content-type", MimeType.HTML)); if (page().nestLevel() != 0) { throw new WebAppException("Error rendering page: nestLevel="+ page().nestLevel()); Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml?rev=1598785&r1=1598784&r2=1598785&view=diff ============================================================================== --- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml (original) +++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml Sat May 31 00:01:20 2014 @@ -309,14 +309,17 @@ Number of times RM tries to connect to ZooKeeper. yarn.resourcemanager.zk-num-retries - 500 + 1000 Retry interval in milliseconds when connecting to ZooKeeper. + When HA is enabled, the value here is NOT used. It is generated + automatically from yarn.resourcemanager.zk-timeout-ms and + yarn.resourcemanager.zk-num-retries. yarn.resourcemanager.zk-retry-interval-ms - 2000 + 1000 @@ -370,6 +373,18 @@ + + Specify the auths to be used for the ACL's specified in both the + yarn.resourcemanager.zk-acl and + yarn.resourcemanager.zk-state-store.root-node.acl properties. This + takes a comma-separated list of authentication mechanisms, each of the + form 'scheme:auth' (the same syntax used for the 'addAuth' command in + the ZK CLI). + + yarn.resourcemanager.zk-auth + + + URI pointing to the location of the FileSystem path where RM state will be stored. This must be supplied when using org.apache.hadoop.yarn.server.resourcemanager.recovery.FileSystemRMStateStore Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/TestSubViews.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/TestSubViews.java?rev=1598785&r1=1598784&r2=1598785&view=diff ============================================================================== --- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/TestSubViews.java (original) +++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/TestSubViews.java Sat May 31 00:01:20 2014 @@ -70,6 +70,6 @@ public class TestSubViews { out.flush(); verify(out).print("sub1 text"); verify(out).print("sub2 text"); - verify(out, times(15)).println(); // test inline transition across views + verify(out, times(16)).println(); // test inline transition across views } } Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/view/TestHtmlPage.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/view/TestHtmlPage.java?rev=1598785&r1=1598784&r2=1598785&view=diff ============================================================================== --- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/view/TestHtmlPage.java (original) +++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/view/TestHtmlPage.java Sat May 31 00:01:20 2014 @@ -22,11 +22,12 @@ import com.google.inject.Injector; import java.io.PrintWriter; +import org.apache.hadoop.yarn.webapp.MimeType; import org.apache.hadoop.yarn.webapp.WebAppException; import org.apache.hadoop.yarn.webapp.test.WebAppTests; import org.apache.hadoop.yarn.webapp.view.HtmlPage; - import org.junit.Test; + import static org.mockito.Mockito.*; public class TestHtmlPage { @@ -53,6 +54,12 @@ public class TestHtmlPage { Injector injector = WebAppTests.testPage(TestView.class); PrintWriter out = injector.getInstance(PrintWriter.class); + // Verify the HTML page has correct meta tags in the header + verify(out).print(" http-equiv=\"X-UA-Compatible\""); + verify(out).print(" content=\"IE=8\""); + verify(out).print(" http-equiv=\"Content-type\""); + verify(out).print(String.format(" content=\"%s\"", MimeType.HTML)); + verify(out).print("test"); verify(out).print(" id=\"testid\""); verify(out).print("test note"); Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml?rev=1598785&r1=1598784&r2=1598785&view=diff ============================================================================== --- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml (original) +++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml Sat May 31 00:01:20 2014 @@ -156,6 +156,10 @@ org.apache.hadoop hadoop-yarn-server-common + + org.fusesource.leveldbjni + leveldbjni-all + @@ -292,6 +296,7 @@ ${basedir}/src/main/proto + yarn_server_nodemanager_recovery.proto yarn_server_nodemanager_service_protos.proto LocalizationProtocol.proto Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java?rev=1598785&r1=1598784&r2=1598785&view=diff ============================================================================== --- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java (original) +++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java Sat May 31 00:01:20 2014 @@ -27,6 +27,7 @@ import org.apache.hadoop.yarn.api.record import org.apache.hadoop.yarn.server.api.records.NodeHealthStatus; import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; +import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService; import org.apache.hadoop.yarn.server.nodemanager.security.NMContainerTokenSecretManager; import org.apache.hadoop.yarn.server.nodemanager.security.NMTokenSecretManagerInNM; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; @@ -67,6 +68,8 @@ public interface Context { ApplicationACLsManager getApplicationACLsManager(); + NMStateStoreService getNMStateStore(); + boolean getDecommissioned(); void setDecommissioned(boolean isDecommissioned); Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java?rev=1598785&r1=1598784&r2=1598785&view=diff ============================================================================== --- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java (original) +++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java Sat May 31 00:01:20 2014 @@ -53,6 +53,9 @@ import org.apache.hadoop.yarn.server.nod import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics; +import org.apache.hadoop.yarn.server.nodemanager.recovery.NMLeveldbStateStoreService; +import org.apache.hadoop.yarn.server.nodemanager.recovery.NMNullStateStoreService; +import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService; import org.apache.hadoop.yarn.server.nodemanager.security.NMContainerTokenSecretManager; import org.apache.hadoop.yarn.server.nodemanager.security.NMTokenSecretManagerInNM; import org.apache.hadoop.yarn.server.nodemanager.webapp.WebServer; @@ -78,6 +81,7 @@ public class NodeManager extends Composi private ContainerManagerImpl containerManager; private NodeStatusUpdater nodeStatusUpdater; private static CompositeServiceShutdownHook nodeManagerShutdownHook; + private NMStateStoreService nmStore = null; private AtomicBoolean isStopping = new AtomicBoolean(false); @@ -115,9 +119,10 @@ public class NodeManager extends Composi protected NMContext createNMContext( NMContainerTokenSecretManager containerTokenSecretManager, - NMTokenSecretManagerInNM nmTokenSecretManager) { + NMTokenSecretManagerInNM nmTokenSecretManager, + NMStateStoreService stateStore) { return new NMContext(containerTokenSecretManager, nmTokenSecretManager, - dirsHandler, aclsManager); + dirsHandler, aclsManager, stateStore); } protected void doSecureLogin() throws IOException { @@ -125,11 +130,8 @@ public class NodeManager extends Composi YarnConfiguration.NM_PRINCIPAL); } - @Override - protected void serviceInit(Configuration conf) throws Exception { - - conf.setBoolean(Dispatcher.DISPATCHER_EXIT_ON_ERROR_KEY, true); - + private void initAndStartRecoveryStore(Configuration conf) + throws IOException { boolean recoveryEnabled = conf.getBoolean( YarnConfiguration.NM_RECOVERY_ENABLED, YarnConfiguration.DEFAULT_NM_RECOVERY_ENABLED); @@ -142,7 +144,36 @@ public class NodeManager extends Composi } Path recoveryRoot = new Path(recoveryDirName); recoveryFs.mkdirs(recoveryRoot, new FsPermission((short)0700)); + nmStore = new NMLeveldbStateStoreService(); + } else { + nmStore = new NMNullStateStoreService(); + } + nmStore.init(conf); + nmStore.start(); + } + + private void stopRecoveryStore() throws IOException { + nmStore.stop(); + if (context.getDecommissioned() && nmStore.canRecover()) { + LOG.info("Removing state store due to decommission"); + Configuration conf = getConfig(); + Path recoveryRoot = new Path( + conf.get(YarnConfiguration.NM_RECOVERY_DIR)); + LOG.info("Removing state store at " + recoveryRoot + + " due to decommission"); + FileSystem recoveryFs = FileSystem.getLocal(conf); + if (!recoveryFs.delete(recoveryRoot, true)) { + LOG.warn("Unable to delete " + recoveryRoot); + } } + } + + @Override + protected void serviceInit(Configuration conf) throws Exception { + + conf.setBoolean(Dispatcher.DISPATCHER_EXIT_ON_ERROR_KEY, true); + + initAndStartRecoveryStore(conf); NMContainerTokenSecretManager containerTokenSecretManager = new NMContainerTokenSecretManager(conf); @@ -171,7 +202,7 @@ public class NodeManager extends Composi dirsHandler = nodeHealthChecker.getDiskHandler(); this.context = createNMContext(containerTokenSecretManager, - nmTokenSecretManager); + nmTokenSecretManager, nmStore); nodeStatusUpdater = createNodeStatusUpdater(context, dispatcher, nodeHealthChecker); @@ -220,6 +251,7 @@ public class NodeManager extends Composi return; } super.serviceStop(); + stopRecoveryStore(); DefaultMetricsSystem.shutdown(); } @@ -272,11 +304,13 @@ public class NodeManager extends Composi private WebServer webServer; private final NodeHealthStatus nodeHealthStatus = RecordFactoryProvider .getRecordFactory(null).newRecordInstance(NodeHealthStatus.class); + private final NMStateStoreService stateStore; private boolean isDecommissioned = false; public NMContext(NMContainerTokenSecretManager containerTokenSecretManager, NMTokenSecretManagerInNM nmTokenSecretManager, - LocalDirsHandlerService dirsHandler, ApplicationACLsManager aclsManager) { + LocalDirsHandlerService dirsHandler, ApplicationACLsManager aclsManager, + NMStateStoreService stateStore) { this.containerTokenSecretManager = containerTokenSecretManager; this.nmTokenSecretManager = nmTokenSecretManager; this.dirsHandler = dirsHandler; @@ -284,6 +318,7 @@ public class NodeManager extends Composi this.nodeHealthStatus.setIsNodeHealthy(true); this.nodeHealthStatus.setHealthReport("Healthy"); this.nodeHealthStatus.setLastHealthReportTime(System.currentTimeMillis()); + this.stateStore = stateStore; } /** @@ -352,6 +387,11 @@ public class NodeManager extends Composi } @Override + public NMStateStoreService getNMStateStore() { + return stateStore; + } + + @Override public boolean getDecommissioned() { return isDecommissioned; } Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java?rev=1598785&r1=1598784&r2=1598785&view=diff ============================================================================== --- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java (original) +++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java Sat May 31 00:01:20 2014 @@ -22,6 +22,7 @@ import static org.apache.hadoop.service. import java.io.IOException; import java.net.InetSocketAddress; +import java.net.URISyntaxException; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Arrays; @@ -116,6 +117,7 @@ import org.apache.hadoop.yarn.server.nod import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitorEventType; import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitorImpl; import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics; +import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService; import org.apache.hadoop.yarn.server.nodemanager.security.authorize.NMPolicyProvider; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; import org.apache.hadoop.yarn.server.utils.BuilderUtils; @@ -218,6 +220,15 @@ public class ContainerManagerImpl extend SHUTDOWN_CLEANUP_SLOP_MS; super.serviceInit(conf); + recover(); + } + + private void recover() throws IOException, URISyntaxException { + NMStateStoreService stateStore = context.getNMStateStore(); + if (stateStore.canRecover()) { + rsrcLocalizationSrvc.recoverLocalizedResources( + stateStore.loadLocalizationState()); + } } protected LogHandler createLogHandler(Configuration conf, Context context, @@ -239,7 +250,7 @@ public class ContainerManagerImpl extend protected ResourceLocalizationService createResourceLocalizationService( ContainerExecutor exec, DeletionService deletionContext) { return new ResourceLocalizationService(this.dispatcher, exec, - deletionContext, dirsHandler); + deletionContext, dirsHandler, context.getNMStateStore()); } protected ContainersLauncher createContainersLauncher(Context context, Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalCacheDirectoryManager.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalCacheDirectoryManager.java?rev=1598785&r1=1598784&r2=1598785&view=diff ============================================================================== --- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalCacheDirectoryManager.java (original) +++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalCacheDirectoryManager.java Sat May 31 00:01:20 2014 @@ -26,6 +26,8 @@ import org.apache.hadoop.conf.Configurat import org.apache.hadoop.fs.Path; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import com.google.common.annotations.VisibleForTesting; + /** * {@link LocalCacheDirectoryManager} is used for managing hierarchical * directories for local cache. It will allow to restrict the number of files in @@ -99,6 +101,57 @@ public class LocalCacheDirectoryManager } } + /** + * Increment the file count for a relative directory within the cache + * + * @param relPath the relative path + */ + public synchronized void incrementFileCountForPath(String relPath) { + relPath = relPath == null ? "" : relPath.trim(); + Directory subDir = knownDirectories.get(relPath); + if (subDir == null) { + int dirnum = Directory.getDirectoryNumber(relPath); + totalSubDirectories = Math.max(dirnum, totalSubDirectories); + subDir = new Directory(dirnum); + nonFullDirectories.add(subDir); + knownDirectories.put(subDir.getRelativePath(), subDir); + } + if (subDir.incrementAndGetCount() >= perDirectoryFileLimit) { + nonFullDirectories.remove(subDir); + } + } + + /** + * Given a path to a directory within a local cache tree return the + * root of the cache directory. + * + * @param path the directory within a cache directory + * @return the local cache directory root or null if not found + */ + public static Path getCacheDirectoryRoot(Path path) { + while (path != null) { + String name = path.getName(); + if (name.length() != 1) { + return path; + } + int dirnum = DIRECTORIES_PER_LEVEL; + try { + dirnum = Integer.parseInt(name, DIRECTORIES_PER_LEVEL); + } catch (NumberFormatException e) { + } + if (dirnum >= DIRECTORIES_PER_LEVEL) { + return path; + } + path = path.getParent(); + } + return path; + } + + @VisibleForTesting + synchronized Directory getDirectory(String relPath) { + return knownDirectories.get(relPath); + } + /* * It limits the number of files and sub directories in the directory to the * limit LocalCacheDirectoryManager#perDirectoryFileLimit. @@ -108,11 +161,9 @@ public class LocalCacheDirectoryManager private final String relativePath; private int fileCount; - public Directory(int directoryNo) { - fileCount = 0; - if (directoryNo == 0) { - relativePath = ""; - } else { + static String getRelativePath(int directoryNo) { + String relativePath = ""; + if (directoryNo > 0) { String tPath = Integer.toString(directoryNo - 1, DIRECTORIES_PER_LEVEL); StringBuffer sb = new StringBuffer(); if (tPath.length() == 1) { @@ -128,6 +179,27 @@ public class LocalCacheDirectoryManager } relativePath = sb.toString(); } + return relativePath; + } + + static int getDirectoryNumber(String relativePath) { + String numStr = relativePath.replace("/", ""); + if (relativePath.isEmpty()) { + return 0; + } + if (numStr.length() > 1) { + // undo step from getRelativePath() to reuse 0th sub directory + String firstChar = Integer.toString( + Integer.parseInt(numStr.substring(0, 1), + DIRECTORIES_PER_LEVEL) + 1, DIRECTORIES_PER_LEVEL); + numStr = firstChar + numStr.substring(1); + } + return Integer.parseInt(numStr, DIRECTORIES_PER_LEVEL) + 1; + } + + public Directory(int directoryNo) { + fileCount = 0; + relativePath = getRelativePath(directoryNo); } public int incrementAndGetCount() { Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTracker.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTracker.java?rev=1598785&r1=1598784&r2=1598785&view=diff ============================================================================== --- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTracker.java (original) +++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTracker.java Sat May 31 00:01:20 2014 @@ -18,15 +18,12 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer; -import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.fs.Path; import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.server.nodemanager.DeletionService; import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceEvent; -import com.google.common.annotations.VisibleForTesting; - /** * Component tracking resources all of the same {@link LocalResourceVisibility} * @@ -34,18 +31,11 @@ import com.google.common.annotations.Vis interface LocalResourcesTracker extends EventHandler, Iterable { - // TODO: Not used at all!! - boolean contains(LocalResourceRequest resource); - boolean remove(LocalizedResource req, DeletionService delService); Path getPathForLocalization(LocalResourceRequest req, Path localDirPath); String getUser(); - long nextUniqueNumber(); - - @VisibleForTesting - @Private LocalizedResource getLocalizedResource(LocalResourceRequest request); } Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java?rev=1598785&r1=1598784&r2=1598785&view=diff ============================================================================== --- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java (original) +++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java Sat May 31 00:01:20 2014 @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer; import java.io.File; +import java.io.IOException; import java.util.Iterator; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -27,14 +28,21 @@ import java.util.regex.Pattern; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; +import org.apache.hadoop.yarn.api.records.impl.pb.LocalResourcePBImpl; import org.apache.hadoop.yarn.event.Dispatcher; +import org.apache.hadoop.yarn.proto.YarnProtos.LocalResourceProto; +import org.apache.hadoop.yarn.proto.YarnServerNodemanagerRecoveryProtos.LocalizedResourceProto; import org.apache.hadoop.yarn.server.nodemanager.DeletionService; import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceEvent; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceEventType; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceRecoveredEvent; import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceReleaseEvent; +import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService; import com.google.common.annotations.VisibleForTesting; @@ -53,6 +61,7 @@ class LocalResourcesTrackerImpl implemen .compile(RANDOM_DIR_REGEX); private final String user; + private final ApplicationId appId; private final Dispatcher dispatcher; private final ConcurrentMap localrsrc; private Configuration conf; @@ -77,17 +86,22 @@ class LocalResourcesTrackerImpl implemen * per APPLICATION, USER and PUBLIC cache. */ private AtomicLong uniqueNumberGenerator = new AtomicLong(9); + private NMStateStoreService stateStore; - public LocalResourcesTrackerImpl(String user, Dispatcher dispatcher, - boolean useLocalCacheDirectoryManager, Configuration conf) { - this(user, dispatcher, + public LocalResourcesTrackerImpl(String user, ApplicationId appId, + Dispatcher dispatcher, boolean useLocalCacheDirectoryManager, + Configuration conf, NMStateStoreService stateStore) { + this(user, appId, dispatcher, new ConcurrentHashMap(), - useLocalCacheDirectoryManager, conf); + useLocalCacheDirectoryManager, conf, stateStore); } - LocalResourcesTrackerImpl(String user, Dispatcher dispatcher, + LocalResourcesTrackerImpl(String user, ApplicationId appId, + Dispatcher dispatcher, ConcurrentMap localrsrc, - boolean useLocalCacheDirectoryManager, Configuration conf) { + boolean useLocalCacheDirectoryManager, Configuration conf, + NMStateStoreService stateStore) { + this.appId = appId; this.user = user; this.dispatcher = dispatcher; this.localrsrc = localrsrc; @@ -98,6 +112,7 @@ class LocalResourcesTrackerImpl implemen new ConcurrentHashMap(); } this.conf = conf; + this.stateStore = stateStore; } /* @@ -119,8 +134,7 @@ class LocalResourcesTrackerImpl implemen if (rsrc != null && (!isResourcePresent(rsrc))) { LOG.info("Resource " + rsrc.getLocalPath() + " is missing, localizing it again"); - localrsrc.remove(req); - decrementFileCountForLocalCacheDirectory(req, rsrc); + removeResource(req); rsrc = null; } if (null == rsrc) { @@ -141,15 +155,102 @@ class LocalResourcesTrackerImpl implemen } break; case LOCALIZATION_FAILED: - decrementFileCountForLocalCacheDirectory(req, null); /* * If resource localization fails then Localized resource will be * removed from local cache. */ - localrsrc.remove(req); + removeResource(req); + break; + case RECOVERED: + if (rsrc != null) { + LOG.warn("Ignoring attempt to recover existing resource " + rsrc); + return; + } + rsrc = recoverResource(req, (ResourceRecoveredEvent) event); + localrsrc.put(req, rsrc); break; } + rsrc.handle(event); + + if (event.getType() == ResourceEventType.LOCALIZED) { + if (rsrc.getLocalPath() != null) { + try { + stateStore.finishResourceLocalization(user, appId, + buildLocalizedResourceProto(rsrc)); + } catch (IOException ioe) { + LOG.error("Error storing resource state for " + rsrc, ioe); + } + } else { + LOG.warn("Resource " + rsrc + " localized without a location"); + } + } + } + + private LocalizedResource recoverResource(LocalResourceRequest req, + ResourceRecoveredEvent event) { + // unique number for a resource is the directory of the resource + Path localDir = event.getLocalPath().getParent(); + long rsrcId = Long.parseLong(localDir.getName()); + + // update ID generator to avoid conflicts with existing resources + while (true) { + long currentRsrcId = uniqueNumberGenerator.get(); + long nextRsrcId = Math.max(currentRsrcId, rsrcId); + if (uniqueNumberGenerator.compareAndSet(currentRsrcId, nextRsrcId)) { + break; + } + } + + incrementFileCountForLocalCacheDirectory(localDir.getParent()); + + return new LocalizedResource(req, dispatcher); + } + + private LocalizedResourceProto buildLocalizedResourceProto( + LocalizedResource rsrc) { + return LocalizedResourceProto.newBuilder() + .setResource(buildLocalResourceProto(rsrc.getRequest())) + .setLocalPath(rsrc.getLocalPath().toString()) + .setSize(rsrc.getSize()) + .build(); + } + + private LocalResourceProto buildLocalResourceProto(LocalResource lr) { + LocalResourcePBImpl lrpb; + if (!(lr instanceof LocalResourcePBImpl)) { + lr = LocalResource.newInstance(lr.getResource(), lr.getType(), + lr.getVisibility(), lr.getSize(), lr.getTimestamp(), + lr.getPattern()); + } + lrpb = (LocalResourcePBImpl) lr; + return lrpb.getProto(); + } + + public void incrementFileCountForLocalCacheDirectory(Path cacheDir) { + if (useLocalCacheDirectoryManager) { + Path cacheRoot = LocalCacheDirectoryManager.getCacheDirectoryRoot( + cacheDir); + if (cacheRoot != null) { + LocalCacheDirectoryManager dir = directoryManagers.get(cacheRoot); + if (dir == null) { + dir = new LocalCacheDirectoryManager(conf); + LocalCacheDirectoryManager otherDir = + directoryManagers.putIfAbsent(cacheRoot, dir); + if (otherDir != null) { + dir = otherDir; + } + } + if (cacheDir.equals(cacheRoot)) { + dir.incrementFileCountForPath(""); + } else { + String dirStr = cacheDir.toUri().getRawPath(); + String rootStr = cacheRoot.toUri().getRawPath(); + dir.incrementFileCountForPath( + dirStr.substring(rootStr.length() + 1)); + } + } + } } /* @@ -217,11 +318,6 @@ class LocalResourcesTrackerImpl implemen } @Override - public boolean contains(LocalResourceRequest resource) { - return localrsrc.containsKey(resource); - } - - @Override public boolean remove(LocalizedResource rem, DeletionService delService) { // current synchronization guaranteed by crude RLS event for cleanup LocalizedResource rsrc = localrsrc.get(rem.getRequest()); @@ -237,16 +333,31 @@ class LocalResourcesTrackerImpl implemen + " with non-zero refcount"); return false; } else { // ResourceState is LOCALIZED or INIT - localrsrc.remove(rem.getRequest()); if (ResourceState.LOCALIZED.equals(rsrc.getState())) { delService.delete(getUser(), getPathToDelete(rsrc.getLocalPath())); } - decrementFileCountForLocalCacheDirectory(rem.getRequest(), rsrc); + removeResource(rem.getRequest()); LOG.info("Removed " + rsrc.getLocalPath() + " from localized cache"); return true; } } + private void removeResource(LocalResourceRequest req) { + LocalizedResource rsrc = localrsrc.remove(req); + decrementFileCountForLocalCacheDirectory(req, rsrc); + if (rsrc != null) { + Path localPath = rsrc.getLocalPath(); + if (localPath != null) { + try { + stateStore.removeLocalizedResource(user, appId, localPath); + } catch (IOException e) { + LOG.error("Unable to remove resource " + rsrc + " from state store", + e); + } + } + } + } + /** * Returns the path up to the random directory component. */ @@ -285,6 +396,7 @@ class LocalResourcesTrackerImpl implemen @Override public Path getPathForLocalization(LocalResourceRequest req, Path localDirPath) { + Path rPath = localDirPath; if (useLocalCacheDirectoryManager && localDirPath != null) { if (!directoryManagers.containsKey(localDirPath)) { @@ -293,7 +405,7 @@ class LocalResourcesTrackerImpl implemen } LocalCacheDirectoryManager dir = directoryManagers.get(localDirPath); - Path rPath = localDirPath; + rPath = localDirPath; String hierarchicalPath = dir.getRelativePathForLocalization(); // For most of the scenarios we will get root path only which // is an empty string @@ -301,21 +413,36 @@ class LocalResourcesTrackerImpl implemen rPath = new Path(localDirPath, hierarchicalPath); } inProgressLocalResourcesMap.put(req, rPath); - return rPath; - } else { - return localDirPath; } - } - @Override - public long nextUniqueNumber() { - return uniqueNumberGenerator.incrementAndGet(); + rPath = new Path(rPath, + Long.toString(uniqueNumberGenerator.incrementAndGet())); + Path localPath = new Path(rPath, req.getPath().getName()); + LocalizedResource rsrc = localrsrc.get(req); + rsrc.setLocalPath(localPath); + LocalResource lr = LocalResource.newInstance(req.getResource(), + req.getType(), req.getVisibility(), req.getSize(), + req.getTimestamp()); + try { + stateStore.startResourceLocalization(user, appId, + ((LocalResourcePBImpl) lr).getProto(), localPath); + } catch (IOException e) { + LOG.error("Unable to record localization start for " + rsrc, e); + } + return rPath; } - @VisibleForTesting - @Private @Override public LocalizedResource getLocalizedResource(LocalResourceRequest request) { return localrsrc.get(request); } -} \ No newline at end of file + + @VisibleForTesting + LocalCacheDirectoryManager getDirectoryManager(Path localDirPath) { + LocalCacheDirectoryManager mgr = null; + if (useLocalCacheDirectoryManager) { + mgr = directoryManagers.get(localDirPath); + } + return mgr; + } +} Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalizedResource.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalizedResource.java?rev=1598785&r1=1598784&r2=1598785&view=diff ============================================================================== --- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalizedResource.java (original) +++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalizedResource.java Sat May 31 00:01:20 2014 @@ -38,6 +38,7 @@ import org.apache.hadoop.yarn.server.nod import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceEventType; import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceFailedLocalizationEvent; import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceLocalizedEvent; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceRecoveredEvent; import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceReleaseEvent; import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceRequestEvent; import org.apache.hadoop.yarn.state.InvalidStateTransitonException; @@ -54,8 +55,8 @@ public class LocalizedResource implement private static final Log LOG = LogFactory.getLog(LocalizedResource.class); - Path localPath; - long size = -1; + volatile Path localPath; + volatile long size = -1; final LocalResourceRequest rsrc; final Dispatcher dispatcher; final StateMachine @@ -76,6 +77,8 @@ public class LocalizedResource implement // From INIT (ref == 0, awaiting req) .addTransition(ResourceState.INIT, ResourceState.DOWNLOADING, ResourceEventType.REQUEST, new FetchResourceTransition()) + .addTransition(ResourceState.INIT, ResourceState.LOCALIZED, + ResourceEventType.RECOVERED, new RecoveredTransition()) // From DOWNLOADING (ref > 0, may be localizing) .addTransition(ResourceState.DOWNLOADING, ResourceState.DOWNLOADING, @@ -157,6 +160,10 @@ public class LocalizedResource implement return localPath; } + public void setLocalPath(Path localPath) { + this.localPath = Path.getPathWithoutSchemeAndAuthority(localPath); + } + public long getTimestamp() { return timestamp.get(); } @@ -234,7 +241,8 @@ public class LocalizedResource implement @Override public void transition(LocalizedResource rsrc, ResourceEvent event) { ResourceLocalizedEvent locEvent = (ResourceLocalizedEvent) event; - rsrc.localPath = locEvent.getLocation(); + rsrc.localPath = + Path.getPathWithoutSchemeAndAuthority(locEvent.getLocation()); rsrc.size = locEvent.getSize(); for (ContainerId container : rsrc.ref) { rsrc.dispatcher.getEventHandler().handle( @@ -291,4 +299,13 @@ public class LocalizedResource implement rsrc.release(relEvent.getContainer()); } } + + private static class RecoveredTransition extends ResourceTransition { + @Override + public void transition(LocalizedResource rsrc, ResourceEvent event) { + ResourceRecoveredEvent recoveredEvent = (ResourceRecoveredEvent) event; + rsrc.localPath = recoveredEvent.getLocalPath(); + rsrc.size = recoveredEvent.getSize(); + } + } } Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java?rev=1598785&r1=1598784&r2=1598785&view=diff ============================================================================== --- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java (original) +++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java Sat May 31 00:01:20 2014 @@ -74,6 +74,7 @@ import org.apache.hadoop.yarn.api.record import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; +import org.apache.hadoop.yarn.api.records.impl.pb.LocalResourcePBImpl; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.event.EventHandler; @@ -81,6 +82,8 @@ import org.apache.hadoop.yarn.exceptions import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.ipc.YarnRPC; +import org.apache.hadoop.yarn.proto.YarnProtos.LocalResourceProto; +import org.apache.hadoop.yarn.proto.YarnServerNodemanagerRecoveryProtos.LocalizedResourceProto; import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor; import org.apache.hadoop.yarn.server.nodemanager.DeletionService; import org.apache.hadoop.yarn.server.nodemanager.DeletionService.FileDeletionTask; @@ -109,10 +112,15 @@ import org.apache.hadoop.yarn.server.nod import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.LocalizerResourceRequestEvent; import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceFailedLocalizationEvent; import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceLocalizedEvent; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceRecoveredEvent; import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceReleaseEvent; import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceRequestEvent; import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.security.LocalizerTokenIdentifier; import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.security.LocalizerTokenSecretManager; +import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService; +import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService.LocalResourceTrackerState; +import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService.RecoveredLocalizationState; +import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService.RecoveredUserResources; import org.apache.hadoop.yarn.server.nodemanager.security.authorize.NMPolicyProvider; import org.apache.hadoop.yarn.server.nodemanager.util.NodeManagerBuilderUtils; import org.apache.hadoop.yarn.util.ConverterUtils; @@ -142,6 +150,7 @@ public class ResourceLocalizationService private RecordFactory recordFactory; private final ScheduledExecutorService cacheCleanup; private LocalizerTokenSecretManager secretManager; + private NMStateStoreService stateStore; private LocalResourcesTracker publicRsrc; @@ -163,7 +172,7 @@ public class ResourceLocalizationService public ResourceLocalizationService(Dispatcher dispatcher, ContainerExecutor exec, DeletionService delService, - LocalDirsHandlerService dirsHandler) { + LocalDirsHandlerService dirsHandler, NMStateStoreService stateStore) { super(ResourceLocalizationService.class.getName()); this.exec = exec; @@ -175,6 +184,7 @@ public class ResourceLocalizationService new ThreadFactoryBuilder() .setNameFormat("ResourceLocalizationService Cache Cleanup") .build()); + this.stateStore = stateStore; } FileContext getLocalFileContext(Configuration conf) { @@ -203,15 +213,17 @@ public class ResourceLocalizationService @Override public void serviceInit(Configuration conf) throws Exception { this.validateConf(conf); - this.publicRsrc = - new LocalResourcesTrackerImpl(null, dispatcher, true, conf); + this.publicRsrc = new LocalResourcesTrackerImpl(null, null, dispatcher, + true, conf, stateStore); this.recordFactory = RecordFactoryProvider.getRecordFactory(conf); try { FileContext lfs = getLocalFileContext(conf); lfs.setUMask(new FsPermission((short)FsPermission.DEFAULT_UMASK)); - cleanUpLocalDir(lfs,delService); + if (!stateStore.canRecover()) { + cleanUpLocalDir(lfs,delService); + } List localDirs = dirsHandler.getLocalDirs(); for (String localDir : localDirs) { @@ -249,6 +261,74 @@ public class ResourceLocalizationService super.serviceInit(conf); } + //Recover localized resources after an NM restart + public void recoverLocalizedResources(RecoveredLocalizationState state) + throws URISyntaxException { + LocalResourceTrackerState trackerState = state.getPublicTrackerState(); + recoverTrackerResources(publicRsrc, trackerState); + + for (Map.Entry userEntry : + state.getUserResources().entrySet()) { + String user = userEntry.getKey(); + RecoveredUserResources userResources = userEntry.getValue(); + trackerState = userResources.getPrivateTrackerState(); + if (!trackerState.isEmpty()) { + LocalResourcesTracker tracker = new LocalResourcesTrackerImpl(user, + null, dispatcher, true, super.getConfig(), stateStore); + LocalResourcesTracker oldTracker = privateRsrc.putIfAbsent(user, + tracker); + if (oldTracker != null) { + tracker = oldTracker; + } + recoverTrackerResources(tracker, trackerState); + } + + for (Map.Entry appEntry : + userResources.getAppTrackerStates().entrySet()) { + trackerState = appEntry.getValue(); + if (!trackerState.isEmpty()) { + ApplicationId appId = appEntry.getKey(); + String appIdStr = ConverterUtils.toString(appId); + LocalResourcesTracker tracker = new LocalResourcesTrackerImpl(user, + appId, dispatcher, false, super.getConfig(), stateStore); + LocalResourcesTracker oldTracker = appRsrc.putIfAbsent(appIdStr, + tracker); + if (oldTracker != null) { + tracker = oldTracker; + } + recoverTrackerResources(tracker, trackerState); + } + } + } + } + + private void recoverTrackerResources(LocalResourcesTracker tracker, + LocalResourceTrackerState state) throws URISyntaxException { + for (LocalizedResourceProto proto : state.getLocalizedResources()) { + LocalResource rsrc = new LocalResourcePBImpl(proto.getResource()); + LocalResourceRequest req = new LocalResourceRequest(rsrc); + LOG.info("Recovering localized resource " + req + " at " + + proto.getLocalPath()); + tracker.handle(new ResourceRecoveredEvent(req, + new Path(proto.getLocalPath()), proto.getSize())); + } + + for (Map.Entry entry : + state.getInProgressResources().entrySet()) { + LocalResource rsrc = new LocalResourcePBImpl(entry.getKey()); + LocalResourceRequest req = new LocalResourceRequest(rsrc); + Path localPath = entry.getValue(); + tracker.handle(new ResourceRecoveredEvent(req, localPath, 0)); + + // delete any in-progress localizations, containers will request again + LOG.info("Deleting in-progress localization for " + req + " at " + + localPath); + tracker.remove(tracker.getLocalizedResource(req), delService); + } + + // TODO: remove untracked directories in local filesystem + } + @Override public LocalizerHeartbeatResponse heartbeat(LocalizerStatus status) { return localizerTracker.processHeartbeat(status); @@ -337,17 +417,10 @@ public class ResourceLocalizationService // 0) Create application tracking structs String userName = app.getUser(); privateRsrc.putIfAbsent(userName, new LocalResourcesTrackerImpl(userName, - dispatcher, true, super.getConfig())); - if (null != appRsrc.putIfAbsent( - ConverterUtils.toString(app.getAppId()), - new LocalResourcesTrackerImpl(app.getUser(), dispatcher, false, super - .getConfig()))) { - LOG.warn("Initializing application " + app + " already present"); - assert false; // TODO: FIXME assert doesn't help - // ^ The condition is benign. Tests should fail and it - // should appear in logs, but it's an internal error - // that should have no effect on applications - } + null, dispatcher, true, super.getConfig(), stateStore)); + String appIdStr = ConverterUtils.toString(app.getAppId()); + appRsrc.putIfAbsent(appIdStr, new LocalResourcesTrackerImpl(app.getUser(), + app.getAppId(), dispatcher, false, super.getConfig(), stateStore)); // 1) Signal container init // // This is handled by the ApplicationImpl state machine and allows @@ -446,18 +519,28 @@ public class ResourceLocalizationService @SuppressWarnings({"unchecked"}) private void handleDestroyApplicationResources(Application application) { - String userName; - String appIDStr; + String userName = application.getUser(); + ApplicationId appId = application.getAppId(); + String appIDStr = application.toString(); LocalResourcesTracker appLocalRsrcsTracker = - appRsrc.remove(ConverterUtils.toString(application.getAppId())); - if (null == appLocalRsrcsTracker) { + appRsrc.remove(ConverterUtils.toString(appId)); + if (appLocalRsrcsTracker != null) { + for (LocalizedResource rsrc : appLocalRsrcsTracker ) { + Path localPath = rsrc.getLocalPath(); + if (localPath != null) { + try { + stateStore.removeLocalizedResource(userName, appId, localPath); + } catch (IOException e) { + LOG.error("Unable to remove resource " + rsrc + " for " + appIDStr + + " from state store", e); + } + } + } + } else { LOG.warn("Removing uninitialized application " + application); } - // TODO: What to do with appLocalRsrcsTracker? // Delete the application directories - userName = application.getUser(); - appIDStr = application.toString(); for (String localDir : dirsHandler.getLocalDirs()) { // Delete the user-owned app-dir @@ -668,19 +751,15 @@ public class ResourceLocalizationService if (rsrc.getState().equals(ResourceState.DOWNLOADING)) { LocalResource resource = request.getResource().getRequest(); try { - Path publicDirDestPath = + Path publicRootPath = dirsHandler.getLocalPathForWrite("." + Path.SEPARATOR + ContainerLocalizer.FILECACHE, ContainerLocalizer.getEstimatedSize(resource), true); - Path hierarchicalPath = - publicRsrc.getPathForLocalization(key, publicDirDestPath); - if (!hierarchicalPath.equals(publicDirDestPath)) { - publicDirDestPath = hierarchicalPath; + Path publicDirDestPath = + publicRsrc.getPathForLocalization(key, publicRootPath); + if (!publicDirDestPath.getParent().equals(publicRootPath)) { DiskChecker.checkDir(new File(publicDirDestPath.toUri().getPath())); } - publicDirDestPath = - new Path(publicDirDestPath, Long.toString(publicRsrc - .nextUniqueNumber())); // explicitly synchronize pending here to avoid future task // completing and being dequeued before pending updated synchronized (pending) { @@ -968,9 +1047,8 @@ public class ResourceLocalizationService Path dirPath = dirsHandler.getLocalPathForWrite(cacheDirectory, ContainerLocalizer.getEstimatedSize(rsrc), false); - dirPath = tracker.getPathForLocalization(new LocalResourceRequest(rsrc), - dirPath); - return new Path (dirPath, Long.toString(tracker.nextUniqueNumber())); + return tracker.getPathForLocalization(new LocalResourceRequest(rsrc), + dirPath); } @Override Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/event/ResourceEventType.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/event/ResourceEventType.java?rev=1598785&r1=1598784&r2=1598785&view=diff ============================================================================== --- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/event/ResourceEventType.java (original) +++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/event/ResourceEventType.java Sat May 31 00:01:20 2014 @@ -31,5 +31,7 @@ public enum ResourceEventType { /** See {@link ResourceReleaseEvent} */ RELEASE, /** See {@link ResourceFailedLocalizationEvent} */ - LOCALIZATION_FAILED + LOCALIZATION_FAILED, + /** See {@link ResourceRecoveredEvent} */ + RECOVERED } Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java?rev=1598785&r1=1598784&r2=1598785&view=diff ============================================================================== --- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java (original) +++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java Sat May 31 00:01:20 2014 @@ -54,6 +54,7 @@ import org.apache.hadoop.yarn.server.nod import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.LogHandler; import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerEvent; import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics; +import org.apache.hadoop.yarn.server.nodemanager.recovery.NMNullStateStoreService; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; public class DummyContainerManager extends ContainerManagerImpl { @@ -75,7 +76,7 @@ public class DummyContainerManager exten protected ResourceLocalizationService createResourceLocalizationService( ContainerExecutor exec, DeletionService deletionContext) { return new ResourceLocalizationService(super.dispatcher, exec, - deletionContext, super.dirsHandler) { + deletionContext, super.dirsHandler, new NMNullStateStoreService()) { @Override public void handle(LocalizationEvent event) { switch (event.getType()) { Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java?rev=1598785&r1=1598784&r2=1598785&view=diff ============================================================================== --- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java (original) +++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java Sat May 31 00:01:20 2014 @@ -44,6 +44,7 @@ import org.apache.hadoop.yarn.server.nod import org.apache.hadoop.yarn.server.nodemanager.containermanager.BaseContainerManagerTest; import org.apache.hadoop.yarn.server.nodemanager.containermanager.TestContainerManager; import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics; +import org.apache.hadoop.yarn.server.nodemanager.recovery.NMNullStateStoreService; import org.apache.hadoop.yarn.server.nodemanager.security.NMContainerTokenSecretManager; import org.apache.hadoop.yarn.server.nodemanager.security.NMTokenSecretManagerInNM; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; @@ -79,7 +80,8 @@ public class TestEventFlow { YarnConfiguration conf = new YarnConfiguration(); Context context = new NMContext(new NMContainerTokenSecretManager(conf), - new NMTokenSecretManagerInNM(), null, null) { + new NMTokenSecretManagerInNM(), null, null, + new NMNullStateStoreService()) { @Override public int getHttpPort() { return 1234; Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java?rev=1598785&r1=1598784&r2=1598785&view=diff ============================================================================== --- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java (original) +++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java Sat May 31 00:01:20 2014 @@ -109,6 +109,36 @@ public class TestNodeManagerShutdown { } @Test + public void testStateStoreRemovalOnDecommission() throws IOException { + final File recoveryDir = new File(basedir, "nm-recovery"); + nm = new TestNodeManager(); + YarnConfiguration conf = createNMConfig(); + conf.setBoolean(YarnConfiguration.NM_RECOVERY_ENABLED, true); + conf.set(YarnConfiguration.NM_RECOVERY_DIR, recoveryDir.getAbsolutePath()); + + // verify state store is not removed on normal shutdown + nm.init(conf); + nm.start(); + Assert.assertTrue(recoveryDir.exists()); + Assert.assertTrue(recoveryDir.isDirectory()); + nm.stop(); + nm = null; + Assert.assertTrue(recoveryDir.exists()); + Assert.assertTrue(recoveryDir.isDirectory()); + + // verify state store is removed on decommissioned shutdown + nm = new TestNodeManager(); + nm.init(conf); + nm.start(); + Assert.assertTrue(recoveryDir.exists()); + Assert.assertTrue(recoveryDir.isDirectory()); + nm.getNMContext().setDecommissioned(true); + nm.stop(); + nm = null; + Assert.assertFalse(recoveryDir.exists()); + } + + @Test public void testKillContainersOnShutdown() throws IOException, YarnException { nm = new TestNodeManager(); Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java?rev=1598785&r1=1598784&r2=1598785&view=diff ============================================================================== --- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java (original) +++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java Sat May 31 00:01:20 2014 @@ -91,6 +91,8 @@ import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import org.apache.hadoop.yarn.server.nodemanager.recovery.NMNullStateStoreService; +import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService; @SuppressWarnings("rawtypes") public class TestNodeStatusUpdater { @@ -1159,7 +1161,8 @@ public class TestNodeStatusUpdater { @Override protected NMContext createNMContext( NMContainerTokenSecretManager containerTokenSecretManager, - NMTokenSecretManagerInNM nmTokenSecretManager) { + NMTokenSecretManagerInNM nmTokenSecretManager, + NMStateStoreService store) { return new MyNMContext(containerTokenSecretManager, nmTokenSecretManager); } @@ -1268,7 +1271,8 @@ public class TestNodeStatusUpdater { public MyNMContext( NMContainerTokenSecretManager containerTokenSecretManager, NMTokenSecretManagerInNM nmTokenSecretManager) { - super(containerTokenSecretManager, nmTokenSecretManager, null, null); + super(containerTokenSecretManager, nmTokenSecretManager, null, null, + new NMNullStateStoreService()); } @Override Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java?rev=1598785&r1=1598784&r2=1598785&view=diff ============================================================================== --- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java (original) +++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java Sat May 31 00:01:20 2014 @@ -64,6 +64,7 @@ import org.apache.hadoop.yarn.server.nod import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationState; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics; +import org.apache.hadoop.yarn.server.nodemanager.recovery.NMNullStateStoreService; import org.apache.hadoop.yarn.server.nodemanager.security.NMContainerTokenSecretManager; import org.apache.hadoop.yarn.server.nodemanager.security.NMTokenSecretManagerInNM; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; @@ -103,7 +104,8 @@ public abstract class BaseContainerManag protected static final int HTTP_PORT = 5412; protected Configuration conf = new YarnConfiguration(); protected Context context = new NMContext(new NMContainerTokenSecretManager( - conf), new NMTokenSecretManagerInNM(), null, new ApplicationACLsManager(conf)) { + conf), new NMTokenSecretManagerInNM(), null, + new ApplicationACLsManager(conf), new NMNullStateStoreService()) { public int getHttpPort() { return HTTP_PORT; }; Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalCacheDirectoryManager.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalCacheDirectoryManager.java?rev=1598785&r1=1598784&r2=1598785&view=diff ============================================================================== --- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalCacheDirectoryManager.java (original) +++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalCacheDirectoryManager.java Sat May 31 00:01:20 2014 @@ -23,6 +23,7 @@ import org.junit.Assert; import org.apache.hadoop.fs.Path; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.LocalCacheDirectoryManager.Directory; import org.junit.Test; public class TestLocalCacheDirectoryManager { @@ -73,7 +74,7 @@ public class TestLocalCacheDirectoryMana conf.set(YarnConfiguration.NM_LOCAL_CACHE_MAX_FILES_PER_DIRECTORY, "1"); Exception e = null; ResourceLocalizationService service = - new ResourceLocalizationService(null, null, null, null); + new ResourceLocalizationService(null, null, null, null, null); try { service.init(conf); } catch (Exception e1) { @@ -109,4 +110,49 @@ public class TestLocalCacheDirectoryMana // first sub directory Assert.assertEquals(firstSubDir, dir.getRelativePathForLocalization()); } + + @Test + public void testDirectoryConversion() { + for (int i = 0; i < 10000; ++i) { + String path = Directory.getRelativePath(i); + Assert.assertEquals("Incorrect conversion for " + i, i, + Directory.getDirectoryNumber(path)); + } + } + + @Test + public void testIncrementFileCountForPath() { + YarnConfiguration conf = new YarnConfiguration(); + conf.setInt(YarnConfiguration.NM_LOCAL_CACHE_MAX_FILES_PER_DIRECTORY, + LocalCacheDirectoryManager.DIRECTORIES_PER_LEVEL + 2); + LocalCacheDirectoryManager mgr = new LocalCacheDirectoryManager(conf); + final String rootPath = ""; + mgr.incrementFileCountForPath(rootPath); + Assert.assertEquals(rootPath, mgr.getRelativePathForLocalization()); + Assert.assertFalse("root dir should be full", + rootPath.equals(mgr.getRelativePathForLocalization())); + // finish filling the other directory + mgr.getRelativePathForLocalization(); + // free up space in the root dir + mgr.decrementFileCountForPath(rootPath); + mgr.decrementFileCountForPath(rootPath); + Assert.assertEquals(rootPath, mgr.getRelativePathForLocalization()); + Assert.assertEquals(rootPath, mgr.getRelativePathForLocalization()); + String otherDir = mgr.getRelativePathForLocalization(); + Assert.assertFalse("root dir should be full", otherDir.equals(rootPath)); + + final String deepDir0 = "d/e/e/p/0"; + final String deepDir1 = "d/e/e/p/1"; + final String deepDir2 = "d/e/e/p/2"; + final String deepDir3 = "d/e/e/p/3"; + mgr.incrementFileCountForPath(deepDir0); + Assert.assertEquals(otherDir, mgr.getRelativePathForLocalization()); + Assert.assertEquals(deepDir0, mgr.getRelativePathForLocalization()); + Assert.assertEquals("total dir count incorrect after increment", + deepDir1, mgr.getRelativePathForLocalization()); + mgr.incrementFileCountForPath(deepDir2); + mgr.incrementFileCountForPath(deepDir1); + mgr.incrementFileCountForPath(deepDir2); + Assert.assertEquals(deepDir3, mgr.getRelativePathForLocalization()); + } }