Return-Path: X-Original-To: apmail-hadoop-yarn-commits-archive@minotaur.apache.org Delivered-To: apmail-hadoop-yarn-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 1E5EFE055 for ; Mon, 18 Mar 2013 11:45:46 +0000 (UTC) Received: (qmail 26585 invoked by uid 500); 18 Mar 2013 11:45:46 -0000 Delivered-To: apmail-hadoop-yarn-commits-archive@hadoop.apache.org Received: (qmail 25472 invoked by uid 500); 18 Mar 2013 11:45:40 -0000 Mailing-List: contact yarn-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: yarn-commits@hadoop.apache.org Delivered-To: mailing list yarn-commits@hadoop.apache.org Received: (qmail 25389 invoked by uid 99); 18 Mar 2013 11:45:38 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 18 Mar 2013 11:45:38 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 18 Mar 2013 11:45:33 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 6A13223888FD; Mon, 18 Mar 2013 11:45:11 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1457716 - in /hadoop/common/branches/HDFS-2802/hadoop-yarn-project: ./ hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/ hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/ hadoop-yarn/h... Date: Mon, 18 Mar 2013 11:45:10 -0000 To: yarn-commits@hadoop.apache.org From: szetszwo@apache.org X-Mailer: svnmailer-1.0.8-patched Message-Id: <20130318114511.6A13223888FD@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: szetszwo Date: Mon Mar 18 11:45:07 2013 New Revision: 1457716 URL: http://svn.apache.org/r1457716 Log: Merge r1455389 through r1457712 from trunk. Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/CHANGES.txt hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/JQueryUI.java hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NavBlock.java hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NodePage.java hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/amfilter/TestAmFilter.java Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/CHANGES.txt?rev=1457716&r1=1457715&r2=1457716&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/CHANGES.txt (original) +++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/CHANGES.txt Mon Mar 18 11:45:07 2013 @@ -66,6 +66,12 @@ Release 2.0.5-beta - UNRELEASED the per-application page are translated to html line breaks. (Omkar Vinit Joshi via vinodkv) + YARN-198. Added a link to RM pages from the NodeManager web app. (Jian He + via vinodkv) + + YARN-237. Refreshing the RM page forgets how many rows I had in my + Datatables (jian he via bobby) + OPTIMIZATIONS BUG FIXES @@ -91,6 +97,9 @@ Release 2.0.5-beta - UNRELEASED YARN-376. Fixes a bug which would prevent the NM knowing about completed containers and applications. (Jason Lowe via sseth) + YARN-196. Nodemanager should be more robust in handling connection failure + to ResourceManager when a cluster is started (Xuan Gong via hitesh) + Release 2.0.4-alpha - UNRELEASED INCOMPATIBLE CHANGES @@ -396,6 +405,8 @@ Release 0.23.7 - UNRELEASED YARN-443. allow OS scheduling priority of NM to be different than the containers it launches (tgraves) + YARN-468. coverage fix for org.apache.hadoop.yarn.server.webproxy.amfilter + (Aleksey Gorshkov via bobby) OPTIMIZATIONS Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java?rev=1457716&r1=1457715&r2=1457716&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java (original) +++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java Mon Mar 18 11:45:07 2013 @@ -621,6 +621,20 @@ public class YarnConfiguration extends C public static final long DEFAULT_NM_PROCESS_KILL_WAIT_MS = 2000; + /** Max time to wait to establish a connection to RM when NM starts + */ + public static final String RESOURCEMANAGER_CONNECT_WAIT_SECS = + NM_PREFIX + "resourcemanager.connect.wait.secs"; + public static final int DEFAULT_RESOURCEMANAGER_CONNECT_WAIT_SECS = + 15*60; + + /** Time interval between each NM attempt to connect to RM + */ + public static final String RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_SECS = + NM_PREFIX + "resourcemanager.connect.retry_interval.secs"; + public static final long DEFAULT_RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_SECS + = 30; + /** * CLASSPATH for YARN applications. A comma-separated list of CLASSPATH * entries Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/JQueryUI.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/JQueryUI.java?rev=1457716&r1=1457715&r2=1457716&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/JQueryUI.java (original) +++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/JQueryUI.java Mon Mar 18 11:45:07 2013 @@ -107,12 +107,21 @@ public class JQueryUI extends HtmlBlock protected void initDataTables(List list) { String defaultInit = "{bJQueryUI: true, sPaginationType: 'full_numbers'}"; + String stateSaveInit = "bStateSave : true, " + + "\"fnStateSave\": function (oSettings, oData) { " + + "sessionStorage.setItem( oSettings.sTableId, JSON.stringify(oData) ); }, " + + "\"fnStateLoad\": function (oSettings) { " + + "return JSON.parse( sessionStorage.getItem(oSettings.sTableId) );}, "; + for (String id : split($(DATATABLES_ID))) { if (Html.isValidId(id)) { String init = $(initID(DATATABLES, id)); if (init.isEmpty()) { init = defaultInit; } + // for inserting stateSaveInit + int pos = init.indexOf('{') + 1; + init = new StringBuffer(init).insert(pos, stateSaveInit).toString(); list.add(join(id,"DataTable = $('#", id, "').dataTable(", init, ").fnSetFilteringDelay(188);")); String postInit = $(postInitID(DATATABLES, id)); @@ -126,9 +135,12 @@ public class JQueryUI extends HtmlBlock String init = $(initSelector(DATATABLES)); if (init.isEmpty()) { init = defaultInit; - } + } + int pos = init.indexOf('{') + 1; + init = new StringBuffer(init).insert(pos, stateSaveInit).toString(); list.add(join(" $('", escapeJavaScript(selector), "').dataTable(", init, - ").fnSetFilteringDelay(288);")); + ").fnSetFilteringDelay(288);")); + } } Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml?rev=1457716&r1=1457715&r2=1457716&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml (original) +++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml Mon Mar 18 11:45:07 2013 @@ -597,6 +597,20 @@ 2000 + + Max time, in seconds, to wait to establish a connection to RM when NM starts. + The NM will shutdown if it cannot connect to RM within the specified max time period. + If the value is set as -1, then NM will retry forever. + yarn.nodemanager.resourcemanager.connect.wait.secs + 900 + + + + Time interval, in seconds, between each NM attempt to connect to RM. + yarn.nodemanager.resourcemanager.connect.retry_interval.secs + 30 + + yarn.nodemanager.aux-services.mapreduce.shuffle.class Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java?rev=1457716&r1=1457715&r2=1457716&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java (original) +++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java Mon Mar 18 11:45:07 2013 @@ -350,7 +350,7 @@ public class NodeManager extends Composi ContainerManagerImpl getContainerManager() { return containerManager; } - + public static void main(String[] args) { Thread.setDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler()); StringUtils.startupShutdownMessage(NodeManager.class, args, LOG); Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java?rev=1457716&r1=1457715&r2=1457716&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java (original) +++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java Mon Mar 18 11:45:07 2013 @@ -151,7 +151,6 @@ public class NodeStatusUpdaterImpl exten YarnConfiguration.DEFAULT_NM_WEBAPP_ADDRESS, YarnConfiguration.DEFAULT_NM_WEBAPP_PORT); try { - // this.hostName = InetAddress.getLocalHost().getCanonicalHostName(); this.httpPort = httpBindAddress.getPort(); // Registration has to be in start so that ContainerManager can get the // perNM tokens needed to authenticate ContainerTokens. @@ -189,15 +188,84 @@ public class NodeStatusUpdaterImpl exten } private void registerWithRM() throws YarnRemoteException { - this.resourceTracker = getRMClient(); - LOG.info("Connecting to ResourceManager at " + this.rmAddress); - - RegisterNodeManagerRequest request = recordFactory.newRecordInstance(RegisterNodeManagerRequest.class); + Configuration conf = getConfig(); + long rmConnectWaitMS = + conf.getInt( + YarnConfiguration.RESOURCEMANAGER_CONNECT_WAIT_SECS, + YarnConfiguration.DEFAULT_RESOURCEMANAGER_CONNECT_WAIT_SECS) + * 1000; + long rmConnectionRetryIntervalMS = + conf.getLong( + YarnConfiguration.RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_SECS, + YarnConfiguration + .DEFAULT_RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_SECS) + * 1000; + + if(rmConnectionRetryIntervalMS < 0) { + throw new YarnException("Invalid Configuration. " + + YarnConfiguration.RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_SECS + + " should not be negative."); + } + + boolean waitForEver = (rmConnectWaitMS == -1000); + + if(! waitForEver) { + if(rmConnectWaitMS < 0) { + throw new YarnException("Invalid Configuration. " + + YarnConfiguration.RESOURCEMANAGER_CONNECT_WAIT_SECS + + " can be -1, but can not be other negative numbers"); + } + + //try connect once + if(rmConnectWaitMS < rmConnectionRetryIntervalMS) { + LOG.warn(YarnConfiguration.RESOURCEMANAGER_CONNECT_WAIT_SECS + + " is smaller than " + + YarnConfiguration.RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_SECS + + ". Only try connect once."); + rmConnectWaitMS = 0; + } + } + + int rmRetryCount = 0; + long waitStartTime = System.currentTimeMillis(); + + RegisterNodeManagerRequest request = + recordFactory.newRecordInstance(RegisterNodeManagerRequest.class); request.setHttpPort(this.httpPort); request.setResource(this.totalResource); request.setNodeId(this.nodeId); - RegistrationResponse regResponse = - this.resourceTracker.registerNodeManager(request).getRegistrationResponse(); + RegistrationResponse regResponse; + + while(true) { + try { + rmRetryCount++; + LOG.info("Connecting to ResourceManager at " + this.rmAddress + + ". current no. of attempts is " + rmRetryCount); + this.resourceTracker = getRMClient(); + regResponse = + this.resourceTracker.registerNodeManager(request) + .getRegistrationResponse(); + break; + } catch(Throwable e) { + LOG.warn("Trying to connect to ResourceManager, " + + "current no. of failed attempts is "+rmRetryCount); + if(System.currentTimeMillis() - waitStartTime < rmConnectWaitMS + || waitForEver) { + try { + LOG.info("Sleeping for " + rmConnectionRetryIntervalMS/1000 + + " seconds before next connection retry to RM"); + Thread.sleep(rmConnectionRetryIntervalMS); + } catch(InterruptedException ex) { + //done nothing + } + } else { + String errorMessage = "Failed to Connect to RM, " + + "no. of failed attempts is "+rmRetryCount; + LOG.error(errorMessage,e); + throw new YarnException(errorMessage,e); + } + } + } // if the Resourcemanager instructs NM to shutdown. if (NodeAction.SHUTDOWN.equals(regResponse.getNodeAction())) { throw new YarnException( Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NavBlock.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NavBlock.java?rev=1457716&r1=1457715&r2=1457716&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NavBlock.java (original) +++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NavBlock.java Mon Mar 18 11:45:07 2013 @@ -18,16 +18,32 @@ package org.apache.hadoop.yarn.server.nodemanager.webapp; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.webapp.YarnWebParams; import org.apache.hadoop.yarn.webapp.view.HtmlBlock; +import com.google.inject.Inject; + public class NavBlock extends HtmlBlock implements YarnWebParams { + private Configuration conf; + + @Inject + public NavBlock(Configuration conf) { + this.conf = conf; + } + @Override protected void render(Block html) { - html + + String RMWebAppURL = YarnConfiguration.getRMWebAppURL(this.conf); + html .div("#nav") - .h3()._("NodeManager")._() // TODO: Problem if no header like this + .h3()._("ResourceManager")._() + .ul() + .li().a(RMWebAppURL, "RM Home")._()._() + .h3()._("NodeManager")._() // TODO: Problem if no header like this .ul() .li() .a(url("node"), "Node Information")._() @@ -37,7 +53,7 @@ public class NavBlock extends HtmlBlock .li() .a(url("allContainers"), "List of Containers")._() ._() - .h3("Tools") + .h3("Tools") .ul() .li().a("/conf", "Configuration")._() .li().a("/logs", "Local logs")._() Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NodePage.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NodePage.java?rev=1457716&r1=1457715&r2=1457716&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NodePage.java (original) +++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NodePage.java Mon Mar 18 11:45:07 2013 @@ -42,7 +42,7 @@ public class NodePage extends NMView { protected void commonPreHead(HTML<_> html) { super.commonPreHead(html); - set(initID(ACCORDION, "nav"), "{autoHeight:false, active:0}"); + set(initID(ACCORDION, "nav"), "{autoHeight:false, active:1}"); } @Override Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java?rev=1457716&r1=1457715&r2=1457716&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java (original) +++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java Mon Mar 18 11:45:07 2013 @@ -267,6 +267,36 @@ public class TestNodeStatusUpdater { } } + private class MyNodeStatusUpdater4 extends NodeStatusUpdaterImpl { + public ResourceTracker resourceTracker = + new MyResourceTracker(this.context); + private Context context; + private final long waitStartTime; + private final long rmStartIntervalMS; + private final boolean rmNeverStart; + + public MyNodeStatusUpdater4(Context context, Dispatcher dispatcher, + NodeHealthCheckerService healthChecker, NodeManagerMetrics metrics, + long rmStartIntervalMS, boolean rmNeverStart) { + super(context, dispatcher, healthChecker, metrics); + this.context = context; + this.waitStartTime = System.currentTimeMillis(); + this.rmStartIntervalMS = rmStartIntervalMS; + this.rmNeverStart = rmNeverStart; + } + + @Override + protected ResourceTracker getRMClient() { + if(System.currentTimeMillis() - waitStartTime <= rmStartIntervalMS + || rmNeverStart) { + throw new YarnException("Faking RM start failure as start " + + "delay timer has not expired."); + } else { + return resourceTracker; + } + } + } + private class MyNodeManager extends NodeManager { private MyNodeStatusUpdater3 nodeStatusUpdater; @@ -580,6 +610,73 @@ public class TestNodeStatusUpdater { + "Recieved SHUTDOWN signal from Resourcemanager ,Registration of NodeManager failed"); } + @Test (timeout = 15000) + public void testNMConnectionToRM() { + final long delta = 1500; + final long connectionWaitSecs = 5; + final long connectionRetryIntervalSecs = 1; + //Waiting for rmStartIntervalMS, RM will be started + final long rmStartIntervalMS = 2*1000; + YarnConfiguration conf = createNMConfig(); + conf.setLong(YarnConfiguration.RESOURCEMANAGER_CONNECT_WAIT_SECS, + connectionWaitSecs); + conf.setLong(YarnConfiguration + .RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_SECS, + connectionRetryIntervalSecs); + + //Test NM try to connect to RM Several times, but finally fail + nm = new NodeManager() { + @Override + protected NodeStatusUpdater createNodeStatusUpdater(Context context, + Dispatcher dispatcher, NodeHealthCheckerService healthChecker) { + NodeStatusUpdater nodeStatusUpdater = new MyNodeStatusUpdater4( + context, dispatcher, healthChecker, metrics, + rmStartIntervalMS, true); + return nodeStatusUpdater; + } + }; + nm.init(conf); + long waitStartTime = System.currentTimeMillis(); + try { + nm.start(); + Assert.fail("NM should have failed to start due to RM connect failure"); + } catch(Exception e) { + Assert.assertTrue("NM should have tried re-connecting to RM during " + + "period of at least " + connectionWaitSecs + " seconds, but " + + "stopped retrying within " + (connectionWaitSecs + delta/1000) + + " seconds", (System.currentTimeMillis() - waitStartTime + >= connectionWaitSecs*1000) && (System.currentTimeMillis() + - waitStartTime < (connectionWaitSecs*1000+delta))); + } + + //Test NM connect to RM, fail at first several attempts, + //but finally success. + nm = new NodeManager() { + @Override + protected NodeStatusUpdater createNodeStatusUpdater(Context context, + Dispatcher dispatcher, NodeHealthCheckerService healthChecker) { + NodeStatusUpdater nodeStatusUpdater = new MyNodeStatusUpdater4( + context, dispatcher, healthChecker, metrics, rmStartIntervalMS, + false); + return nodeStatusUpdater; + } + }; + + nm.init(conf); + waitStartTime = System.currentTimeMillis(); + try { + nm.start(); + } catch (Exception ex){ + Assert.fail("NM should have started successfully " + + "after connecting to RM."); + } + Assert.assertTrue("NM should have connected to RM within " + delta/1000 + +" seconds of RM starting up.", + (System.currentTimeMillis() - waitStartTime >= rmStartIntervalMS) + && (System.currentTimeMillis() - waitStartTime + < (rmStartIntervalMS+delta))); + } + /** * Verifies that if for some reason NM fails to start ContainerManager RPC * server, RM is oblivious to NM's presence. The behaviour is like this Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/amfilter/TestAmFilter.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/amfilter/TestAmFilter.java?rev=1457716&r1=1457715&r2=1457716&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/amfilter/TestAmFilter.java (original) +++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/amfilter/TestAmFilter.java Mon Mar 18 11:45:07 2013 @@ -19,41 +19,39 @@ package org.apache.hadoop.yarn.server.webproxy.amfilter; import java.io.IOException; -import java.util.Collections; -import java.util.Enumeration; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; +import java.util.*; import java.util.concurrent.atomic.AtomicBoolean; -import javax.servlet.Filter; -import javax.servlet.FilterChain; -import javax.servlet.FilterConfig; -import javax.servlet.ServletContext; -import javax.servlet.ServletException; -import javax.servlet.ServletRequest; -import javax.servlet.ServletResponse; +import javax.servlet.*; +import javax.servlet.http.Cookie; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; -import junit.framework.Assert; +import static junit.framework.Assert.*; +import org.apache.hadoop.yarn.server.webproxy.WebAppProxyServlet; +import org.glassfish.grizzly.servlet.HttpServletResponseImpl; import org.junit.Test; import org.mockito.Mockito; +/** + * Test AmIpFilter. Requests to a no declared hosts should has way through + * proxy. Another requests can be filtered with (without) user name. + * + */ +public class TestAmFilter { -public class TestAmFilter { - - private String proxyHost = "bogushost.com"; + private String proxyHost = "localhost"; private String proxyUri = "http://bogus"; + private String doFilterRequest; + private AmIpServletRequestWrapper servletWrapper; private class TestAmIpFilter extends AmIpFilter { private Set proxyAddresses = null; protected Set getProxyAddresses() { - if(proxyAddresses == null) { + if (proxyAddresses == null) { proxyAddresses = new HashSet(); } proxyAddresses.add(proxyHost); @@ -61,12 +59,10 @@ public class TestAmFilter { } } - private static class DummyFilterConfig implements FilterConfig { final Map map; - - DummyFilterConfig(Map map) { + DummyFilterConfig(Map map) { this.map = map; } @@ -74,22 +70,24 @@ public class TestAmFilter { public String getFilterName() { return "dummy"; } + @Override public String getInitParameter(String arg0) { return map.get(arg0); } + @Override public Enumeration getInitParameterNames() { return Collections.enumeration(map.keySet()); } + @Override public ServletContext getServletContext() { return null; } } - - @Test + @Test(timeout = 5000) public void filterNullCookies() throws Exception { HttpServletRequest request = Mockito.mock(HttpServletRequest.class); @@ -97,13 +95,12 @@ public class TestAmFilter { Mockito.when(request.getRemoteAddr()).thenReturn(proxyHost); HttpServletResponse response = Mockito.mock(HttpServletResponse.class); - final AtomicBoolean invoked = new AtomicBoolean(); FilterChain chain = new FilterChain() { @Override - public void doFilter(ServletRequest servletRequest, ServletResponse servletResponse) - throws IOException, ServletException { + public void doFilter(ServletRequest servletRequest, + ServletResponse servletResponse) throws IOException, ServletException { invoked.set(true); } }; @@ -115,7 +112,93 @@ public class TestAmFilter { Filter filter = new TestAmIpFilter(); filter.init(conf); filter.doFilter(request, response, chain); - Assert.assertTrue(invoked.get()); + assertTrue(invoked.get()); filter.destroy(); } + + /** + * Test AmIpFilter + */ + @Test(timeout = 1000) + public void testFilter() throws Exception { + Map params = new HashMap(); + params.put(AmIpFilter.PROXY_HOST, proxyHost); + params.put(AmIpFilter.PROXY_URI_BASE, proxyUri); + FilterConfig config = new DummyFilterConfig(params); + + // dummy filter + FilterChain chain = new FilterChain() { + @Override + public void doFilter(ServletRequest servletRequest, + ServletResponse servletResponse) throws IOException, ServletException { + doFilterRequest = servletRequest.getClass().getName(); + if (servletRequest instanceof AmIpServletRequestWrapper) { + servletWrapper = (AmIpServletRequestWrapper) servletRequest; + + } + } + }; + AmIpFilter testFilter = new AmIpFilter(); + testFilter.init(config); + + HttpServletResponseForTest response = new HttpServletResponseForTest(); + // Test request should implements HttpServletRequest + + ServletRequest failRequest = Mockito.mock(ServletRequest.class); + try { + testFilter.doFilter(failRequest, response, chain); + fail(); + } catch (ServletException e) { + assertEquals("This filter only works for HTTP/HTTPS", e.getMessage()); + } + + // request with HttpServletRequest + HttpServletRequest request = Mockito.mock(HttpServletRequest.class); + Mockito.when(request.getRemoteAddr()).thenReturn("redirect"); + Mockito.when(request.getRequestURI()).thenReturn("/redirect"); + testFilter.doFilter(request, response, chain); + // address "redirect" is not in host list + assertEquals("http://bogus/redirect", response.getRedirect()); + // "127.0.0.1" contains in host list. Without cookie + Mockito.when(request.getRemoteAddr()).thenReturn("127.0.0.1"); + testFilter.doFilter(request, response, chain); + + assertTrue(doFilterRequest + .contains("javax.servlet.http.HttpServletRequest")); + // cookie added + Cookie[] cookies = new Cookie[1]; + cookies[0] = new Cookie(WebAppProxyServlet.PROXY_USER_COOKIE_NAME, "user"); + + Mockito.when(request.getCookies()).thenReturn(cookies); + testFilter.doFilter(request, response, chain); + + assertEquals( + "org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpServletRequestWrapper", + doFilterRequest); + // request contains principal from cookie + assertEquals("user", servletWrapper.getUserPrincipal().getName()); + assertEquals("user", servletWrapper.getRemoteUser()); + assertFalse(servletWrapper.isUserInRole("")); + + } + + private class HttpServletResponseForTest extends HttpServletResponseImpl { + String redirectLocation = ""; + + public String getRedirect() { + return redirectLocation; + } + + @Override + public void sendRedirect(String location) throws IOException { + redirectLocation = location; + } + + @Override + public String encodeRedirectURL(String url) { + return url; + } + + } + }