Return-Path: X-Original-To: apmail-hadoop-yarn-commits-archive@minotaur.apache.org Delivered-To: apmail-hadoop-yarn-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id E8974D9E3 for ; Tue, 16 Oct 2012 00:05:52 +0000 (UTC) Received: (qmail 31005 invoked by uid 500); 16 Oct 2012 00:05:52 -0000 Delivered-To: apmail-hadoop-yarn-commits-archive@hadoop.apache.org Received: (qmail 30972 invoked by uid 500); 16 Oct 2012 00:05:52 -0000 Mailing-List: contact yarn-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: yarn-commits@hadoop.apache.org Delivered-To: mailing list yarn-commits@hadoop.apache.org Received: (qmail 30964 invoked by uid 99); 16 Oct 2012 00:05:52 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 16 Oct 2012 00:05:52 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 16 Oct 2012 00:05:48 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 7B1902388CA4; Tue, 16 Oct 2012 00:04:22 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1398581 [4/9] - in /hadoop/common/branches/MR-3902/hadoop-yarn-project: ./ hadoop-yarn/ hadoop-yarn/bin/ hadoop-yarn/conf/ hadoop-yarn/hadoop-yarn-api/ hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ hadoop-yarn/hadoo... Date: Tue, 16 Oct 2012 00:04:13 -0000 To: yarn-commits@hadoop.apache.org From: sseth@apache.org X-Mailer: svnmailer-1.0.8-patched Message-Id: <20121016000422.7B1902388CA4@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Modified: hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java?rev=1398581&r1=1398580&r2=1398581&view=diff ============================================================================== --- hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java (original) +++ hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java Tue Oct 16 00:02:55 2012 @@ -21,10 +21,10 @@ package org.apache.hadoop.yarn.server.no import static org.apache.hadoop.yarn.service.Service.STATE.STARTED; import java.io.IOException; -import java.net.InetAddress; import java.net.InetSocketAddress; import java.nio.ByteBuffer; import java.util.Map; +import java.util.Set; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -96,7 +96,6 @@ import org.apache.hadoop.yarn.server.nod import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics; import org.apache.hadoop.yarn.server.nodemanager.security.authorize.NMPolicyProvider; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; -import org.apache.hadoop.yarn.server.security.ContainerTokenSecretManager; import org.apache.hadoop.yarn.service.CompositeService; import org.apache.hadoop.yarn.service.Service; import org.apache.hadoop.yarn.service.ServiceStateChangeListener; @@ -110,14 +109,12 @@ public class ContainerManagerImpl extend final Context context; private final ContainersMonitor containersMonitor; private Server server; - private InetAddress resolvedAddress = null; private final ResourceLocalizationService rsrcLocalizationSrvc; private final ContainersLauncher containersLauncher; private final AuxServices auxiliaryServices; private final NodeManagerMetrics metrics; private final NodeStatusUpdater nodeStatusUpdater; - private ContainerTokenSecretManager containerTokenSecretManager; private final RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null); @@ -129,8 +126,7 @@ public class ContainerManagerImpl extend public ContainerManagerImpl(Context context, ContainerExecutor exec, DeletionService deletionContext, NodeStatusUpdater nodeStatusUpdater, - NodeManagerMetrics metrics, ContainerTokenSecretManager - containerTokenSecretManager, ApplicationACLsManager aclsManager, + NodeManagerMetrics metrics, ApplicationACLsManager aclsManager, LocalDirsHandlerService dirsHandler) { super(ContainerManagerImpl.class.getName()); this.context = context; @@ -149,7 +145,6 @@ public class ContainerManagerImpl extend addService(containersLauncher); this.nodeStatusUpdater = nodeStatusUpdater; - this.containerTokenSecretManager = containerTokenSecretManager; this.aclsManager = aclsManager; // Start configurable services @@ -232,7 +227,7 @@ public class ContainerManagerImpl extend server = rpc.getServer(ContainerManager.class, this, initialAddress, conf, - this.containerTokenSecretManager, + this.context.getContainerTokenSecretManager(), conf.getInt(YarnConfiguration.NM_CONTAINER_MGR_THREAD_COUNT, YarnConfiguration.DEFAULT_NM_CONTAINER_MGR_THREAD_COUNT)); @@ -267,56 +262,77 @@ public class ContainerManagerImpl extend super.stop(); } + // Get the remoteUGI corresponding to the api call. + private UserGroupInformation getRemoteUgi(String containerIDStr) + throws YarnRemoteException { + UserGroupInformation remoteUgi; + try { + remoteUgi = UserGroupInformation.getCurrentUser(); + } catch (IOException e) { + String msg = "Cannot obtain the user-name for containerId: " + + containerIDStr + ". Got exception: " + + StringUtils.stringifyException(e); + LOG.warn(msg); + throw RPCUtil.getRemoteException(msg); + } + return remoteUgi; + } + + // Obtain the needed ContainerTokenIdentifier from the remote-UGI. RPC layer + // currently sets only the required id, but iterate through anyways just to + // be sure. + private ContainerTokenIdentifier selectContainerTokenIdentifier( + UserGroupInformation remoteUgi) { + Set tokenIdentifiers = remoteUgi.getTokenIdentifiers(); + ContainerTokenIdentifier resultId = null; + for (TokenIdentifier id : tokenIdentifiers) { + if (id instanceof ContainerTokenIdentifier) { + resultId = (ContainerTokenIdentifier) id; + break; + } + } + return resultId; + } + /** * Authorize the request. * - * @param containerID + * @param containerIDStr * of the container * @param launchContext * passed if verifying the startContainer, null otherwise. + * @param remoteUgi + * ugi corresponding to the remote end making the api-call * @throws YarnRemoteException */ - private void authorizeRequest(ContainerId containerID, - ContainerLaunchContext launchContext) throws YarnRemoteException { + private void authorizeRequest(String containerIDStr, + ContainerLaunchContext launchContext, UserGroupInformation remoteUgi) + throws YarnRemoteException { if (!UserGroupInformation.isSecurityEnabled()) { return; } - String containerIDStr = containerID.toString(); - - UserGroupInformation remoteUgi; - try { - remoteUgi = UserGroupInformation.getCurrentUser(); - } catch (IOException e) { - String msg = "Cannot obtain the user-name for containerId: " - + containerIDStr + ". Got exception: " - + StringUtils.stringifyException(e); - LOG.warn(msg); - throw RPCUtil.getRemoteException(msg); - } - boolean unauthorized = false; - StringBuilder messageBuilder = new StringBuilder( - "Unauthorized request to start container. "); + StringBuilder messageBuilder = + new StringBuilder("Unauthorized request to start container. "); if (!remoteUgi.getUserName().equals(containerIDStr)) { unauthorized = true; messageBuilder.append("\nExpected containerId: " + remoteUgi.getUserName() + " Found: " + containerIDStr); - } - - if (launchContext != null) { - - // Verify other things for startContainer() request. + } else if (launchContext != null) { + // Verify other things also for startContainer() request. if (LOG.isDebugEnabled()) { - LOG.debug("Number of TokenIdentifiers in the UGI from RPC: " - + remoteUgi.getTokenIdentifiers().size()); + LOG.debug("Number of TokenIdentifiers in the UGI from RPC: " + + remoteUgi.getTokenIdentifiers().size()); } - // We must and should get only one TokenIdentifier from the RPC. - ContainerTokenIdentifier tokenId = (ContainerTokenIdentifier) remoteUgi - .getTokenIdentifiers().iterator().next(); + + // Get the tokenId from the remote user ugi + ContainerTokenIdentifier tokenId = + selectContainerTokenIdentifier(remoteUgi); + if (tokenId == null) { unauthorized = true; messageBuilder @@ -324,6 +340,23 @@ public class ContainerManagerImpl extend + containerIDStr); } else { + // Is the container coming in with correct user-name? + if (!tokenId.getApplicationSubmitter().equals(launchContext.getUser())) { + unauthorized = true; + messageBuilder.append("\n Expected user-name " + + tokenId.getApplicationSubmitter() + " but found " + + launchContext.getUser()); + } + + // Is the container being relaunched? Or RPC layer let startCall with + // tokens generated off old-secret through? + if (!this.context.getContainerTokenSecretManager() + .isValidStartContainerRequest(tokenId)) { + unauthorized = true; + messageBuilder.append("\n Attempt to relaunch the same " + + "container with id " + containerIDStr + "."); + } + // Ensure the token is not expired. // Token expiry is not checked for stopContainer/getContainerStatus if (tokenId.getExpiryTimeStamp() < System.currentTimeMillis()) { @@ -348,7 +381,7 @@ public class ContainerManagerImpl extend throw RPCUtil.getRemoteException(msg); } } - + /** * Start a container on this NodeManager. */ @@ -359,10 +392,13 @@ public class ContainerManagerImpl extend ContainerLaunchContext launchContext = request.getContainerLaunchContext(); ContainerId containerID = launchContext.getContainerId(); - authorizeRequest(containerID, launchContext); + String containerIDStr = containerID.toString(); + + UserGroupInformation remoteUgi = getRemoteUgi(containerIDStr); + authorizeRequest(containerIDStr, launchContext, remoteUgi); - LOG.info("Start request for " + launchContext.getContainerId() - + " by user " + launchContext.getUser()); + LOG.info("Start request for " + containerIDStr + " by user " + + launchContext.getUser()); // //////////// Parse credentials ByteBuffer tokens = launchContext.getContainerTokens(); @@ -394,14 +430,14 @@ public class ContainerManagerImpl extend AuditConstants.START_CONTAINER, "ContainerManagerImpl", "Container already running on this node!", applicationID, containerID); - throw RPCUtil.getRemoteException("Container " + containerID + throw RPCUtil.getRemoteException("Container " + containerIDStr + " already is running on this node!!"); } // Create the application Application application = new ApplicationImpl(dispatcher, this.aclsManager, - launchContext.getUser(), applicationID, credentials, context); + launchContext.getUser(), applicationID, credentials, context); if (null == context.getApplications().putIfAbsent(applicationID, application)) { LOG.info("Creating a new application reference for app " @@ -414,6 +450,12 @@ public class ContainerManagerImpl extend // TODO: Validate the request dispatcher.getEventHandler().handle( new ApplicationContainerInitEvent(container)); + if (UserGroupInformation.isSecurityEnabled()) { + ContainerTokenIdentifier tokenId = + selectContainerTokenIdentifier(remoteUgi); + this.context.getContainerTokenSecretManager().startContainerSuccessful( + tokenId); + } NMAuditLogger.logSuccess(launchContext.getUser(), AuditConstants.START_CONTAINER, "ContainerManageImpl", @@ -438,8 +480,12 @@ public class ContainerManagerImpl extend throws YarnRemoteException { ContainerId containerID = request.getContainerId(); + String containerIDStr = containerID.toString(); + // TODO: Only the container's owner can kill containers today. - authorizeRequest(containerID, null); + + UserGroupInformation remoteUgi = getRemoteUgi(containerIDStr); + authorizeRequest(containerIDStr, null, remoteUgi); StopContainerResponse response = recordFactory.newRecordInstance(StopContainerResponse.class); @@ -476,10 +522,14 @@ public class ContainerManagerImpl extend GetContainerStatusRequest request) throws YarnRemoteException { ContainerId containerID = request.getContainerId(); + String containerIDStr = containerID.toString(); + // TODO: Only the container's owner can get containers' status today. - authorizeRequest(containerID, null); - LOG.info("Getting container-status for " + containerID); + UserGroupInformation remoteUgi = getRemoteUgi(containerIDStr); + authorizeRequest(containerIDStr, null, remoteUgi); + + LOG.info("Getting container-status for " + containerIDStr); Container container = this.context.getContainers().get(containerID); if (container != null) { ContainerStatus containerStatus = container.cloneAndGetContainerStatus(); @@ -490,7 +540,7 @@ public class ContainerManagerImpl extend return response; } - throw RPCUtil.getRemoteException("Container " + containerID + throw RPCUtil.getRemoteException("Container " + containerIDStr + " is not handled by this NodeManager"); } Modified: hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java?rev=1398581&r1=1398580&r2=1398581&view=diff ============================================================================== --- hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java (original) +++ hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java Tue Oct 16 00:02:55 2012 @@ -28,8 +28,9 @@ import java.util.concurrent.locks.Reentr import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.security.Credentials; -import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.records.ApplicationAccessType; +import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.logaggregation.ContainerLogsRetentionPolicy; @@ -42,6 +43,7 @@ import org.apache.hadoop.yarn.server.nod import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceLocalizationService; import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ApplicationLocalizationEvent; import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.LocalizationEventType; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.logaggregation.LogAggregationService; import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerAppFinishedEvent; import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerAppStartedEvent; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; @@ -365,6 +367,10 @@ public class ApplicationImpl implements @Override public void transition(ApplicationImpl app, ApplicationEvent event) { + // Inform the ContainerTokenSecretManager + if (UserGroupInformation.isSecurityEnabled()) { + app.context.getContainerTokenSecretManager().appFinished(app.appId); + } // Inform the logService app.dispatcher.getEventHandler().handle( new LogHandlerAppFinishedEvent(app.appId)); Modified: hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java?rev=1398581&r1=1398580&r2=1398581&view=diff ============================================================================== --- hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java (original) +++ hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java Tue Oct 16 00:02:55 2012 @@ -209,6 +209,7 @@ public class ContainerLocalizer { } switch (rsrc.getType()) { case ARCHIVE: + case PATTERN: return 5 * rsrc.getSize(); case FILE: default: Modified: hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourceRequest.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourceRequest.java?rev=1398581&r1=1398580&r2=1398581&view=diff ============================================================================== --- hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourceRequest.java (original) +++ hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourceRequest.java Tue Oct 16 00:02:55 2012 @@ -34,6 +34,7 @@ public class LocalResourceRequest private final long timestamp; private final LocalResourceType type; private final LocalResourceVisibility visibility; + private final String pattern; /** * Wrap API resource to match against cache of localized resources. @@ -45,22 +46,28 @@ public class LocalResourceRequest this(ConverterUtils.getPathFromYarnURL(resource.getResource()), resource.getTimestamp(), resource.getType(), - resource.getVisibility()); + resource.getVisibility(), + resource.getPattern()); } LocalResourceRequest(Path loc, long timestamp, LocalResourceType type, - LocalResourceVisibility visibility) { + LocalResourceVisibility visibility, String pattern) { this.loc = loc; this.timestamp = timestamp; this.type = type; this.visibility = visibility; + this.pattern = pattern; } @Override public int hashCode() { - return loc.hashCode() ^ + int hash = loc.hashCode() ^ (int)((timestamp >>> 32) ^ timestamp) * type.hashCode(); + if(pattern != null) { + hash = hash ^ pattern.hashCode(); + } + return hash; } @Override @@ -72,9 +79,14 @@ public class LocalResourceRequest return false; } final LocalResourceRequest other = (LocalResourceRequest) o; + String pattern = getPattern(); + String otherPattern = other.getPattern(); + boolean patternEquals = (pattern == null && otherPattern == null) || + (pattern != null && otherPattern != null && pattern.equals(otherPattern)); return getPath().equals(other.getPath()) && getTimestamp() == other.getTimestamp() && - getType() == other.getType(); + getType() == other.getType() && + patternEquals; } @Override @@ -87,6 +99,19 @@ public class LocalResourceRequest ret = (int)(getTimestamp() - other.getTimestamp()); if (0 == ret) { ret = getType().ordinal() - other.getType().ordinal(); + if (0 == ret) { + String pattern = getPattern(); + String otherPattern = other.getPattern(); + if (pattern == null && otherPattern == null) { + ret = 0; + } else if (pattern == null) { + ret = -1; + } else if (otherPattern == null) { + ret = 1; + } else { + ret = pattern.compareTo(otherPattern); + } + } } } return ret; @@ -122,6 +147,11 @@ public class LocalResourceRequest } @Override + public String getPattern() { + return pattern; + } + + @Override public void setResource(URL resource) { throw new UnsupportedOperationException(); } @@ -145,14 +175,20 @@ public class LocalResourceRequest public void setVisibility(LocalResourceVisibility visibility) { throw new UnsupportedOperationException(); } - + + @Override + public void setPattern(String pattern) { + throw new UnsupportedOperationException(); + } + @Override public String toString() { StringBuilder sb = new StringBuilder(); sb.append("{ "); sb.append(getPath().toString()).append(", "); sb.append(getTimestamp()).append(", "); - sb.append(getType()).append(" }"); + sb.append(getType()).append(", "); + sb.append(getPattern()).append(" }"); return sb.toString(); } } Modified: hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalizedResource.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalizedResource.java?rev=1398581&r1=1398580&r2=1398581&view=diff ============================================================================== --- hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalizedResource.java (original) +++ hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalizedResource.java Tue Oct 16 00:02:55 2012 @@ -219,7 +219,8 @@ public class LocalizedResource implement ContainerId container = ctxt.getContainerId(); rsrc.ref.add(container); rsrc.dispatcher.getEventHandler().handle( - new LocalizerResourceRequestEvent(rsrc, req.getVisibility(), ctxt)); + new LocalizerResourceRequestEvent(rsrc, req.getVisibility(), ctxt, + req.getLocalResourceRequest().getPattern())); } } Modified: hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java?rev=1398581&r1=1398580&r2=1398581&view=diff ============================================================================== --- hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java (original) +++ hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java Tue Oct 16 00:02:55 2012 @@ -181,6 +181,7 @@ public class ResourceLocalizationService try { // TODO queue deletions here, rather than NM init? FileContext lfs = getLocalFileContext(conf); + lfs.setUMask(new FsPermission((short)FsPermission.DEFAULT_UMASK)); List localDirs = dirsHandler.getLocalDirs(); for (String localDir : localDirs) { // $local/usercache @@ -751,6 +752,7 @@ public class ResourceLocalizationService next.setTimestamp(nextRsrc.getTimestamp()); next.setType(nextRsrc.getType()); next.setVisibility(evt.getVisibility()); + next.setPattern(evt.getPattern()); scheduled.put(nextRsrc, evt); return next; } Modified: hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/event/LocalizerResourceRequestEvent.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/event/LocalizerResourceRequestEvent.java?rev=1398581&r1=1398580&r2=1398581&view=diff ============================================================================== --- hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/event/LocalizerResourceRequestEvent.java (original) +++ hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/event/LocalizerResourceRequestEvent.java Tue Oct 16 00:02:55 2012 @@ -32,14 +32,16 @@ public class LocalizerResourceRequestEve private final LocalizerContext context; private final LocalizedResource resource; private final LocalResourceVisibility vis; + private final String pattern; public LocalizerResourceRequestEvent(LocalizedResource resource, - LocalResourceVisibility vis, LocalizerContext context) { + LocalResourceVisibility vis, LocalizerContext context, String pattern) { super(LocalizerEventType.REQUEST_RESOURCE_LOCALIZATION, ConverterUtils.toString(context.getContainerId())); this.vis = vis; this.context = context; this.resource = resource; + this.pattern = pattern; } public LocalizedResource getResource() { @@ -54,4 +56,8 @@ public class LocalizerResourceRequestEve return vis; } + public String getPattern() { + return pattern; + } + } Modified: hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregator.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregator.java?rev=1398581&r1=1398580&r2=1398581&view=diff ============================================================================== --- hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregator.java (original) +++ hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregator.java Tue Oct 16 00:02:55 2012 @@ -26,7 +26,4 @@ public interface AppLogAggregator extend boolean wasContainerSuccessful); void finishLogAggregation(); - - void join(); - } Modified: hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java?rev=1398581&r1=1398580&r2=1398581&view=diff ============================================================================== --- hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java (original) +++ hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java Tue Oct 16 00:02:55 2012 @@ -137,6 +137,9 @@ public class AppLogAggregatorImpl implem try { doAppLogAggregation(); } finally { + if (!this.appAggregationFinished.get()) { + LOG.warn("Aggregation did not complete for application " + appId); + } this.appAggregationFinished.set(true); } } @@ -155,6 +158,7 @@ public class AppLogAggregatorImpl implem } } catch (InterruptedException e) { LOG.warn("PendingContainers queue is interrupted"); + this.appFinishing.set(true); } } @@ -197,6 +201,7 @@ public class AppLogAggregatorImpl implem this.dispatcher.getEventHandler().handle( new ApplicationEvent(this.appId, ApplicationEventType.APPLICATION_LOG_HANDLING_FINISHED)); + this.appAggregationFinished.set(true); } private Path getRemoteNodeTmpLogFileForApp() { @@ -250,21 +255,4 @@ public class AppLogAggregatorImpl implem LOG.info("Application just finished : " + this.applicationId); this.appFinishing.set(true); } - - @Override - public void join() { - // Aggregation service is finishing - this.finishLogAggregation(); - - while (!this.appAggregationFinished.get()) { - LOG.info("Waiting for aggregation to complete for " - + this.applicationId); - try { - Thread.sleep(THREAD_SLEEP_TIME); - } catch (InterruptedException e) { - LOG.warn("Join interrupted. Some logs may not have been aggregated!!"); - break; - } - } - } } Modified: hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java?rev=1398581&r1=1398580&r2=1398581&view=diff ============================================================================== --- hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java (original) +++ hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java Tue Oct 16 00:02:55 2012 @@ -25,6 +25,7 @@ import java.util.concurrent.ConcurrentHa import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -35,8 +36,6 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.security.token.Token; -import org.apache.hadoop.security.token.TokenIdentifier; import org.apache.hadoop.yarn.YarnException; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationAccessType; @@ -137,11 +136,33 @@ public class LogAggregationService exten @Override public synchronized void stop() { LOG.info(this.getName() + " waiting for pending aggregation during exit"); - for (AppLogAggregator appLogAggregator : this.appLogAggregators.values()) { - appLogAggregator.join(); - } + stopAggregators(); super.stop(); } + + private void stopAggregators() { + threadPool.shutdown(); + // politely ask to finish + for (AppLogAggregator aggregator : appLogAggregators.values()) { + aggregator.finishLogAggregation(); + } + while (!threadPool.isTerminated()) { // wait for all threads to finish + for (ApplicationId appId : appLogAggregators.keySet()) { + LOG.info("Waiting for aggregation to complete for " + appId); + } + try { + if (!threadPool.awaitTermination(30, TimeUnit.SECONDS)) { + threadPool.shutdownNow(); // send interrupt to hurry them along + } + } catch (InterruptedException e) { + LOG.warn("Aggregation stop interrupted!"); + break; + } + } + for (ApplicationId appId : appLogAggregators.keySet()) { + LOG.warn("Some logs may not have been aggregated for " + appId); + } + } private void verifyAndCreateRemoteLogDir(Configuration conf) { // Checking the existance of the TLD @@ -293,10 +314,7 @@ public class LogAggregationService exten final UserGroupInformation userUgi = UserGroupInformation.createRemoteUser(user); if (credentials != null) { - for (Token token : credentials - .getAllTokens()) { - userUgi.addToken(token); - } + userUgi.addCredentials(credentials); } // New application @@ -312,9 +330,13 @@ public class LogAggregationService exten try { // Create the app dir createAppDir(user, appId, userUgi); - } catch (YarnException e) { + } catch (Exception e) { + appLogAggregators.remove(appId); closeFileSystems(userUgi); - throw e; + if (!(e instanceof YarnException)) { + e = new YarnException(e); + } + throw (YarnException)e; } Modified: hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/NonAggregatingLogHandler.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/NonAggregatingLogHandler.java?rev=1398581&r1=1398580&r2=1398581&view=diff ============================================================================== --- hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/NonAggregatingLogHandler.java (original) +++ hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/NonAggregatingLogHandler.java Tue Oct 16 00:02:55 2012 @@ -79,16 +79,18 @@ public class NonAggregatingLogHandler ex @Override public void stop() { - sched.shutdown(); - boolean isShutdown = false; - try { - isShutdown = sched.awaitTermination(10, TimeUnit.SECONDS); - } catch (InterruptedException e) { - sched.shutdownNow(); - isShutdown = true; - } - if (!isShutdown) { - sched.shutdownNow(); + if (sched != null) { + sched.shutdown(); + boolean isShutdown = false; + try { + isShutdown = sched.awaitTermination(10, TimeUnit.SECONDS); + } catch (InterruptedException e) { + sched.shutdownNow(); + isShutdown = true; + } + if (!isShutdown) { + sched.shutdownNow(); + } } super.stop(); } Modified: hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java?rev=1398581&r1=1398580&r2=1398581&view=diff ============================================================================== --- hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java (original) +++ hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java Tue Oct 16 00:02:55 2012 @@ -37,7 +37,7 @@ import org.apache.hadoop.yarn.server.nod import org.apache.hadoop.yarn.server.nodemanager.Context; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerKillEvent; import org.apache.hadoop.yarn.service.AbstractService; -import org.apache.hadoop.yarn.util.ProcfsBasedProcessTree; +import org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree; import org.apache.hadoop.yarn.util.ResourceCalculatorPlugin; import com.google.common.base.Preconditions; @@ -60,6 +60,8 @@ public class ContainersMonitorImpl exten private final Dispatcher eventDispatcher; private final Context context; private ResourceCalculatorPlugin resourceCalculatorPlugin; + private Configuration conf; + private Class processTreeClass; private long maxVmemAllottedForContainers = DISABLED_MEMORY_LIMIT; private long maxPmemAllottedForContainers = DISABLED_MEMORY_LIMIT; @@ -96,6 +98,11 @@ public class ContainersMonitorImpl exten ResourceCalculatorPlugin.getResourceCalculatorPlugin(clazz, conf); LOG.info(" Using ResourceCalculatorPlugin : " + this.resourceCalculatorPlugin); + processTreeClass = conf.getClass(YarnConfiguration.NM_CONTAINER_MON_PROCESS_TREE, null, + ResourceCalculatorProcessTree.class); + this.conf = conf; + LOG.info(" Using ResourceCalculatorProcessTree : " + + this.processTreeClass); long totalPhysicalMemoryOnNM = DISABLED_MEMORY_LIMIT; if (this.resourceCalculatorPlugin != null) { @@ -140,7 +147,7 @@ public class ContainersMonitorImpl exten /** * Is the total physical memory check enabled? - * + * * @return true if total physical memory check is enabled. */ boolean isPhysicalMemoryCheckEnabled() { @@ -149,7 +156,7 @@ public class ContainersMonitorImpl exten /** * Is the total virtual memory check enabled? - * + * * @return true if total virtual memory check is enabled. */ boolean isVirtualMemoryCheckEnabled() { @@ -157,12 +164,16 @@ public class ContainersMonitorImpl exten } private boolean isEnabled() { - if (!ProcfsBasedProcessTree.isAvailable()) { - LOG.info("ProcessTree implementation is missing on this system. " - + this.getClass().getName() + " is disabled."); - return false; + if (resourceCalculatorPlugin == null) { + LOG.info("ResourceCalculatorPlugin is unavailable on this system. " + + this.getClass().getName() + " is disabled."); + return false; + } + if (ResourceCalculatorProcessTree.getResourceCalculatorProcessTree("0", processTreeClass, conf) == null) { + LOG.info("ResourceCalculatorProcessTree is unavailable on this system. " + + this.getClass().getName() + " is disabled."); + return false; } - if (!(isPhysicalMemoryCheckEnabled() || isVirtualMemoryCheckEnabled())) { LOG.info("Neither virutal-memory nor physical-memory monitoring is " + "needed. Not running the monitor-thread"); @@ -196,12 +207,12 @@ public class ContainersMonitorImpl exten private static class ProcessTreeInfo { private ContainerId containerId; private String pid; - private ProcfsBasedProcessTree pTree; + private ResourceCalculatorProcessTree pTree; private long vmemLimit; private long pmemLimit; public ProcessTreeInfo(ContainerId containerId, String pid, - ProcfsBasedProcessTree pTree, long vmemLimit, long pmemLimit) { + ResourceCalculatorProcessTree pTree, long vmemLimit, long pmemLimit) { this.containerId = containerId; this.pid = pid; this.pTree = pTree; @@ -221,11 +232,11 @@ public class ContainersMonitorImpl exten this.pid = pid; } - public ProcfsBasedProcessTree getProcessTree() { + public ResourceCalculatorProcessTree getProcessTree() { return this.pTree; } - public void setProcessTree(ProcfsBasedProcessTree pTree) { + public void setProcessTree(ResourceCalculatorProcessTree pTree) { this.pTree = pTree; } @@ -245,20 +256,20 @@ public class ContainersMonitorImpl exten /** * Check whether a container's process tree's current memory usage is over * limit. - * + * * When a java process exec's a program, it could momentarily account for * double the size of it's memory, because the JVM does a fork()+exec() * which at fork time creates a copy of the parent's memory. If the * monitoring thread detects the memory used by the container tree at the * same instance, it could assume it is over limit and kill the tree, for no * fault of the process itself. - * + * * We counter this problem by employing a heuristic check: - if a process * tree exceeds the memory limit by more than twice, it is killed * immediately - if a process tree has processes older than the monitoring * interval exceeding the memory limit by even 1 time, it is killed. Else it * is given the benefit of doubt to lie around for one more iteration. - * + * * @param containerId * Container Id for the container tree * @param currentMemUsage @@ -295,7 +306,7 @@ public class ContainersMonitorImpl exten } // method provided just for easy testing purposes - boolean isProcessTreeOverLimit(ProcfsBasedProcessTree pTree, + boolean isProcessTreeOverLimit(ResourceCalculatorProcessTree pTree, String containerId, long limit) { long currentMemUsage = pTree.getCumulativeVmem(); // as processes begin with an age 1, we want to see if there are processes @@ -370,9 +381,8 @@ public class ContainersMonitorImpl exten LOG.debug("Tracking ProcessTree " + pId + " for the first time"); - ProcfsBasedProcessTree pt = - new ProcfsBasedProcessTree(pId, - ContainerExecutor.isSetsidAvailable); + ResourceCalculatorProcessTree pt = + ResourceCalculatorProcessTree.getResourceCalculatorProcessTree(pId, processTreeClass, conf); ptInfo.setPid(pId); ptInfo.setProcessTree(pt); } @@ -385,7 +395,7 @@ public class ContainersMonitorImpl exten LOG.debug("Constructing ProcessTree for : PID = " + pId + " ContainerId = " + containerId); - ProcfsBasedProcessTree pTree = ptInfo.getProcessTree(); + ResourceCalculatorProcessTree pTree = ptInfo.getProcessTree(); pTree = pTree.getProcessTree(); // get the updated process-tree ptInfo.setProcessTree(pTree); // update ptInfo with proces-tree of // updated state @@ -471,7 +481,7 @@ public class ContainersMonitorImpl exten private String formatErrorMessage(String memTypeExceeded, long currentVmemUsage, long vmemLimit, long currentPmemUsage, long pmemLimit, - String pId, ContainerId containerId, ProcfsBasedProcessTree pTree) { + String pId, ContainerId containerId, ResourceCalculatorProcessTree pTree) { return String.format("Container [pid=%s,containerID=%s] is running beyond %s memory limits. ", pId, containerId, memTypeExceeded) + Modified: hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java?rev=1398581&r1=1398580&r2=1398581&view=diff ============================================================================== --- hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java (original) +++ hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java Tue Oct 16 00:02:55 2012 @@ -27,8 +27,6 @@ import org.apache.commons.logging.LogFac import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.yarn.api.records.ContainerId; -import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; -import org.apache.hadoop.yarn.server.security.ContainerTokenSecretManager; import org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManagerImpl; import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application; import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEvent; @@ -50,6 +48,7 @@ import org.apache.hadoop.yarn.server.nod import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.LogHandler; import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerEvent; import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics; +import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; public class DummyContainerManager extends ContainerManagerImpl { @@ -59,11 +58,10 @@ public class DummyContainerManager exten public DummyContainerManager(Context context, ContainerExecutor exec, DeletionService deletionContext, NodeStatusUpdater nodeStatusUpdater, NodeManagerMetrics metrics, - ContainerTokenSecretManager containerTokenSecretManager, ApplicationACLsManager applicationACLsManager, LocalDirsHandlerService dirsHandler) { super(context, exec, deletionContext, nodeStatusUpdater, metrics, - containerTokenSecretManager, applicationACLsManager, dirsHandler); + applicationACLsManager, dirsHandler); } @Override Modified: hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDefaultContainerExecutor.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDefaultContainerExecutor.java?rev=1398581&r1=1398580&r2=1398581&view=diff ============================================================================== --- hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDefaultContainerExecutor.java (original) +++ hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDefaultContainerExecutor.java Tue Oct 16 00:02:55 2012 @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.server.nodemanager; +import java.io.FileNotFoundException; import java.io.InputStream; import java.io.IOException; import java.net.InetSocketAddress; @@ -26,8 +27,11 @@ import java.util.EnumSet; import java.util.List; import java.util.Random; +import junit.framework.Assert; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.AbstractFileSystem; +import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileContext; @@ -38,6 +42,7 @@ import org.apache.hadoop.fs.permission.F import org.apache.hadoop.io.DataInputBuffer; import org.apache.hadoop.io.DataOutputBuffer; import org.apache.hadoop.util.Progressable; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer; import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.FakeFSDataInputStream; import static org.apache.hadoop.fs.CreateFlag.*; @@ -91,11 +96,16 @@ public class TestDefaultContainerExecuto } */ + private static final Path BASE_TMP_PATH = new Path("target", + TestDefaultContainerExecutor.class.getSimpleName()); + @AfterClass public static void deleteTmpFiles() throws IOException { FileContext lfs = FileContext.getLocalFSFileContext(); - lfs.delete(new Path("target", - TestDefaultContainerExecutor.class.getSimpleName()), true); + try { + lfs.delete(BASE_TMP_PATH, true); + } catch (FileNotFoundException e) { + } } byte[] createTmpFile(Path dst, Random r, int len) @@ -116,6 +126,71 @@ public class TestDefaultContainerExecuto return bytes; } + @Test + public void testDirPermissions() throws Exception { + deleteTmpFiles(); + + final String user = "somebody"; + final String appId = "app_12345_123"; + final FsPermission userCachePerm = new FsPermission( + DefaultContainerExecutor.USER_PERM); + final FsPermission appCachePerm = new FsPermission( + DefaultContainerExecutor.APPCACHE_PERM); + final FsPermission fileCachePerm = new FsPermission( + DefaultContainerExecutor.FILECACHE_PERM); + final FsPermission appDirPerm = new FsPermission( + DefaultContainerExecutor.APPDIR_PERM); + final FsPermission logDirPerm = new FsPermission( + DefaultContainerExecutor.LOGDIR_PERM); + List localDirs = new ArrayList(); + localDirs.add(new Path(BASE_TMP_PATH, "localDirA").toString()); + localDirs.add(new Path(BASE_TMP_PATH, "localDirB").toString()); + List logDirs = new ArrayList(); + logDirs.add(new Path(BASE_TMP_PATH, "logDirA").toString()); + logDirs.add(new Path(BASE_TMP_PATH, "logDirB").toString()); + + Configuration conf = new Configuration(); + conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "077"); + FileContext lfs = FileContext.getLocalFSFileContext(conf); + DefaultContainerExecutor executor = new DefaultContainerExecutor(lfs); + executor.init(); + + try { + executor.createUserLocalDirs(localDirs, user); + executor.createUserCacheDirs(localDirs, user); + executor.createAppDirs(localDirs, user, appId); + + for (String dir : localDirs) { + FileStatus stats = lfs.getFileStatus( + new Path(new Path(dir, ContainerLocalizer.USERCACHE), user)); + Assert.assertEquals(userCachePerm, stats.getPermission()); + } + + for (String dir : localDirs) { + Path userCachePath = new Path( + new Path(dir, ContainerLocalizer.USERCACHE), user); + Path appCachePath = new Path(userCachePath, + ContainerLocalizer.APPCACHE); + FileStatus stats = lfs.getFileStatus(appCachePath); + Assert.assertEquals(appCachePerm, stats.getPermission()); + stats = lfs.getFileStatus( + new Path(userCachePath, ContainerLocalizer.FILECACHE)); + Assert.assertEquals(fileCachePerm, stats.getPermission()); + stats = lfs.getFileStatus(new Path(appCachePath, appId)); + Assert.assertEquals(appDirPerm, stats.getPermission()); + } + + executor.createAppLogDirs(appId, logDirs); + + for (String dir : logDirs) { + FileStatus stats = lfs.getFileStatus(new Path(dir, appId)); + Assert.assertEquals(logDirPerm, stats.getPermission()); + } + } finally { + deleteTmpFiles(); + } + } + // @Test // public void testInit() throws IOException, InterruptedException { // Configuration conf = new Configuration(); Modified: hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDirectoryCollection.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDirectoryCollection.java?rev=1398581&r1=1398580&r2=1398581&view=diff ============================================================================== --- hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDirectoryCollection.java (original) +++ hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDirectoryCollection.java Tue Oct 16 00:02:55 2012 @@ -23,7 +23,13 @@ import java.io.IOException; import java.util.List; import java.util.ListIterator; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeys; +import org.apache.hadoop.fs.FileContext; +import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsPermission; import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; @@ -65,4 +71,37 @@ public class TestDirectoryCollection { // Verify no ConcurrentModification is thrown li.next(); } + + @Test + public void testCreateDirectories() throws IOException { + Configuration conf = new Configuration(); + conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "077"); + FileContext localFs = FileContext.getLocalFSFileContext(conf); + + String dirA = new File(testDir, "dirA").getPath(); + String dirB = new File(dirA, "dirB").getPath(); + String dirC = new File(testDir, "dirC").getPath(); + Path pathC = new Path(dirC); + FsPermission permDirC = new FsPermission((short)0710); + + localFs.mkdir(pathC, null, true); + localFs.setPermission(pathC, permDirC); + + String[] dirs = { dirA, dirB, dirC }; + DirectoryCollection dc = new DirectoryCollection(dirs); + FsPermission defaultPerm = FsPermission.getDefault() + .applyUMask(new FsPermission((short)FsPermission.DEFAULT_UMASK)); + boolean createResult = dc.createNonExistentDirs(localFs, defaultPerm); + Assert.assertTrue(createResult); + + FileStatus status = localFs.getFileStatus(new Path(dirA)); + Assert.assertEquals("local dir parent not created with proper permissions", + defaultPerm, status.getPermission()); + status = localFs.getFileStatus(new Path(dirB)); + Assert.assertEquals("local dir not created with proper permissions", + defaultPerm, status.getPermission()); + status = localFs.getFileStatus(pathC); + Assert.assertEquals("existing local directory permissions modified", + permDirC, status.getPermission()); + } } Modified: hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java?rev=1398581&r1=1398580&r2=1398581&view=diff ============================================================================== --- hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java (original) +++ hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java Tue Oct 16 00:02:55 2012 @@ -36,12 +36,12 @@ import org.apache.hadoop.yarn.event.Asyn import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; -import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; -import org.apache.hadoop.yarn.server.security.ContainerTokenSecretManager; import org.apache.hadoop.yarn.server.api.ResourceTracker; import org.apache.hadoop.yarn.server.nodemanager.NodeManager.NMContext; import org.apache.hadoop.yarn.server.nodemanager.containermanager.BaseContainerManagerTest; import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics; +import org.apache.hadoop.yarn.server.nodemanager.security.NMContainerTokenSecretManager; +import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; import org.junit.Test; public class TestEventFlow { @@ -69,9 +69,9 @@ public class TestEventFlow { localLogDir.mkdir(); remoteLogDir.mkdir(); - Context context = new NMContext(); - YarnConfiguration conf = new YarnConfiguration(); + Context context = new NMContext(new NMContainerTokenSecretManager(conf)); + conf.set(YarnConfiguration.NM_LOCAL_DIRS, localDir.getAbsolutePath()); conf.set(YarnConfiguration.NM_LOG_DIRS, localLogDir.getAbsolutePath()); conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, @@ -86,10 +86,8 @@ public class TestEventFlow { healthChecker.init(conf); LocalDirsHandlerService dirsHandler = healthChecker.getDiskHandler(); NodeManagerMetrics metrics = NodeManagerMetrics.create(); - ContainerTokenSecretManager containerTokenSecretManager = - new ContainerTokenSecretManager(conf); NodeStatusUpdater nodeStatusUpdater = - new NodeStatusUpdaterImpl(context, dispatcher, healthChecker, metrics, containerTokenSecretManager) { + new NodeStatusUpdaterImpl(context, dispatcher, healthChecker, metrics) { @Override protected ResourceTracker getRMClient() { return new LocalRMInterface(); @@ -101,10 +99,9 @@ public class TestEventFlow { } }; - DummyContainerManager containerManager = new DummyContainerManager( - context, exec, del, nodeStatusUpdater, metrics, - containerTokenSecretManager, new ApplicationACLsManager(conf), - dirsHandler); + DummyContainerManager containerManager = + new DummyContainerManager(context, exec, del, nodeStatusUpdater, + metrics, new ApplicationACLsManager(conf), dirsHandler); containerManager.init(conf); containerManager.start(); Modified: hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNMAuditLogger.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNMAuditLogger.java?rev=1398581&r1=1398580&r2=1398581&view=diff ============================================================================== --- hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNMAuditLogger.java (original) +++ hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNMAuditLogger.java Tue Oct 16 00:02:55 2012 @@ -17,37 +17,29 @@ */ package org.apache.hadoop.yarn.server.nodemanager; +import static junit.framework.Assert.assertEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + import java.net.InetAddress; import java.net.InetSocketAddress; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.ipc.RPC; import org.apache.hadoop.ipc.Server; import org.apache.hadoop.ipc.TestRPC.TestImpl; import org.apache.hadoop.ipc.TestRPC.TestProtocol; +import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; -import org.apache.hadoop.yarn.server.nodemanager.NMAuditLogger; -import org.apache.hadoop.yarn.server.nodemanager.NMAuditLogger.AuditConstants; import org.apache.hadoop.yarn.server.nodemanager.NMAuditLogger.Keys; -import org.apache.hadoop.yarn.server.security.ContainerTokenSecretManager; - -import org.apache.hadoop.net.NetUtils; - -import static org.mockito.Mockito.*; -import static junit.framework.Assert.*; -import org.junit.After; import org.junit.Before; import org.junit.Test; - /** * Tests {@link NMAuditLogger}. */ public class TestNMAuditLogger { - private static final Log LOG = LogFactory.getLog(TestNMAuditLogger.class); private static final String USER = "test"; private static final String OPERATION = "oper"; private static final String TARGET = "tgt"; @@ -212,8 +204,10 @@ public class TestNMAuditLogger { public void testNMAuditLoggerWithIP() throws Exception { Configuration conf = new Configuration(); // start the IPC server - Server server = RPC.getServer(TestProtocol.class, - new MyTestRPCServer(), "0.0.0.0", 0, 5, true, conf, null); + Server server = new RPC.Builder(conf).setProtocol(TestProtocol.class) + .setInstance(new MyTestRPCServer()).setBindAddress("0.0.0.0") + .setPort(0).setNumHandlers(5).setVerbose(true).build(); + server.start(); InetSocketAddress addr = NetUtils.getConnectAddress(server); Modified: hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java?rev=1398581&r1=1398580&r2=1398581&view=diff ============================================================================== --- hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java (original) +++ hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java Tue Oct 16 00:02:55 2012 @@ -65,7 +65,6 @@ import org.apache.hadoop.yarn.server.nod import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerImpl; import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; -import org.apache.hadoop.yarn.server.security.ContainerTokenSecretManager; import org.apache.hadoop.yarn.service.Service; import org.apache.hadoop.yarn.service.Service.STATE; import org.apache.hadoop.yarn.util.BuilderUtils; @@ -234,10 +233,8 @@ public class TestNodeStatusUpdater { private Context context; public MyNodeStatusUpdater(Context context, Dispatcher dispatcher, - NodeHealthCheckerService healthChecker, NodeManagerMetrics metrics, - ContainerTokenSecretManager containerTokenSecretManager) { - super(context, dispatcher, healthChecker, metrics, - containerTokenSecretManager); + NodeHealthCheckerService healthChecker, NodeManagerMetrics metrics) { + super(context, dispatcher, healthChecker, metrics); this.context = context; } @@ -252,10 +249,8 @@ public class TestNodeStatusUpdater { private Context context; public MyNodeStatusUpdater3(Context context, Dispatcher dispatcher, - NodeHealthCheckerService healthChecker, NodeManagerMetrics metrics, - ContainerTokenSecretManager containerTokenSecretManager) { - super(context, dispatcher, healthChecker, metrics, - containerTokenSecretManager); + NodeHealthCheckerService healthChecker, NodeManagerMetrics metrics) { + super(context, dispatcher, healthChecker, metrics); this.context = context; this.resourceTracker = new MyResourceTracker3(this.context); } @@ -266,7 +261,7 @@ public class TestNodeStatusUpdater { } @Override - protected boolean isSecurityEnabled() { + protected boolean isTokenKeepAliveEnabled(Configuration conf) { return true; } } @@ -276,11 +271,9 @@ public class TestNodeStatusUpdater { private MyNodeStatusUpdater3 nodeStatusUpdater; @Override protected NodeStatusUpdater createNodeStatusUpdater(Context context, - Dispatcher dispatcher, NodeHealthCheckerService healthChecker, - ContainerTokenSecretManager containerTokenSecretManager) { + Dispatcher dispatcher, NodeHealthCheckerService healthChecker) { this.nodeStatusUpdater = - new MyNodeStatusUpdater3(context, dispatcher, healthChecker, metrics, - containerTokenSecretManager); + new MyNodeStatusUpdater3(context, dispatcher, healthChecker, metrics); return this.nodeStatusUpdater; } @@ -398,10 +391,9 @@ public class TestNodeStatusUpdater { nm = new NodeManager() { @Override protected NodeStatusUpdater createNodeStatusUpdater(Context context, - Dispatcher dispatcher, NodeHealthCheckerService healthChecker, - ContainerTokenSecretManager containerTokenSecretManager) { + Dispatcher dispatcher, NodeHealthCheckerService healthChecker) { return new MyNodeStatusUpdater(context, dispatcher, healthChecker, - metrics, containerTokenSecretManager); + metrics); } }; @@ -528,11 +520,9 @@ public class TestNodeStatusUpdater { nm = new NodeManager() { @Override protected NodeStatusUpdater createNodeStatusUpdater(Context context, - Dispatcher dispatcher, NodeHealthCheckerService healthChecker, - ContainerTokenSecretManager containerTokenSecretManager) { + Dispatcher dispatcher, NodeHealthCheckerService healthChecker) { MyNodeStatusUpdater nodeStatusUpdater = new MyNodeStatusUpdater( - context, dispatcher, healthChecker, metrics, - containerTokenSecretManager); + context, dispatcher, healthChecker, metrics); MyResourceTracker2 myResourceTracker2 = new MyResourceTracker2(); myResourceTracker2.registerNodeAction = NodeAction.SHUTDOWN; nodeStatusUpdater.resourceTracker = myResourceTracker2; @@ -556,22 +546,19 @@ public class TestNodeStatusUpdater { nm = new NodeManager() { @Override protected NodeStatusUpdater createNodeStatusUpdater(Context context, - Dispatcher dispatcher, NodeHealthCheckerService healthChecker, - ContainerTokenSecretManager containerTokenSecretManager) { + Dispatcher dispatcher, NodeHealthCheckerService healthChecker) { return new MyNodeStatusUpdater(context, dispatcher, healthChecker, - metrics, containerTokenSecretManager); + metrics); } @Override protected ContainerManagerImpl createContainerManager(Context context, ContainerExecutor exec, DeletionService del, NodeStatusUpdater nodeStatusUpdater, - ContainerTokenSecretManager containerTokenSecretManager, ApplicationACLsManager aclsManager, LocalDirsHandlerService diskhandler) { - return new ContainerManagerImpl(context, exec, del, - nodeStatusUpdater, metrics, containerTokenSecretManager, - aclsManager, diskhandler) { + return new ContainerManagerImpl(context, exec, del, nodeStatusUpdater, + metrics, aclsManager, diskhandler) { @Override public void start() { // Simulating failure of starting RPC server @@ -654,11 +641,9 @@ public class TestNodeStatusUpdater { return new NodeManager() { @Override protected NodeStatusUpdater createNodeStatusUpdater(Context context, - Dispatcher dispatcher, NodeHealthCheckerService healthChecker, - ContainerTokenSecretManager containerTokenSecretManager) { + Dispatcher dispatcher, NodeHealthCheckerService healthChecker) { MyNodeStatusUpdater myNodeStatusUpdater = new MyNodeStatusUpdater( - context, dispatcher, healthChecker, metrics, - containerTokenSecretManager); + context, dispatcher, healthChecker, metrics); MyResourceTracker2 myResourceTracker2 = new MyResourceTracker2(); myResourceTracker2.heartBeatNodeAction = nodeHeartBeatAction; myNodeStatusUpdater.resourceTracker = myResourceTracker2; Modified: hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java?rev=1398581&r1=1398580&r2=1398581&view=diff ============================================================================== --- hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java (original) +++ hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java Tue Oct 16 00:02:55 2012 @@ -54,8 +54,8 @@ import org.apache.hadoop.yarn.server.nod import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application; import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationState; import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics; +import org.apache.hadoop.yarn.server.nodemanager.security.NMContainerTokenSecretManager; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; -import org.apache.hadoop.yarn.server.security.ContainerTokenSecretManager; import org.apache.hadoop.yarn.service.Service.STATE; import org.junit.After; import org.junit.Before; @@ -70,8 +70,6 @@ public abstract class BaseContainerManag protected static File localLogDir; protected static File remoteLogDir; protected static File tmpDir; - protected ContainerTokenSecretManager containerTokenSecretManager = - new ContainerTokenSecretManager(new Configuration()); protected final NodeManagerMetrics metrics = NodeManagerMetrics.create(); @@ -93,7 +91,8 @@ public abstract class BaseContainerManag .getLog(BaseContainerManagerTest.class); protected Configuration conf = new YarnConfiguration(); - protected Context context = new NMContext(); + protected Context context = new NMContext(new NMContainerTokenSecretManager( + conf)); protected ContainerExecutor exec; protected DeletionService delSrvc; protected String user = "nobody"; @@ -101,7 +100,7 @@ public abstract class BaseContainerManag protected LocalDirsHandlerService dirsHandler; protected NodeStatusUpdater nodeStatusUpdater = new NodeStatusUpdaterImpl( - context, new AsyncDispatcher(), null, metrics, this.containerTokenSecretManager) { + context, new AsyncDispatcher(), null, metrics) { @Override protected ResourceTracker getRMClient() { return new LocalRMInterface(); @@ -155,9 +154,9 @@ public abstract class BaseContainerManag nodeHealthChecker = new NodeHealthCheckerService(); nodeHealthChecker.init(conf); dirsHandler = nodeHealthChecker.getDiskHandler(); - containerManager = new ContainerManagerImpl(context, exec, delSrvc, - nodeStatusUpdater, metrics, this.containerTokenSecretManager, - new ApplicationACLsManager(conf), dirsHandler); + containerManager = + new ContainerManagerImpl(context, exec, delSrvc, nodeStatusUpdater, + metrics, new ApplicationACLsManager(conf), dirsHandler); containerManager.init(conf); } Modified: hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java?rev=1398581&r1=1398580&r2=1398581&view=diff ============================================================================== --- hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java (original) +++ hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java Tue Oct 16 00:02:55 2012 @@ -51,14 +51,13 @@ import org.apache.hadoop.yarn.api.record import org.apache.hadoop.yarn.api.records.URL; import org.apache.hadoop.yarn.exceptions.YarnRemoteException; import org.apache.hadoop.yarn.server.nodemanager.CMgrCompletedAppsEvent; -import org.apache.hadoop.yarn.server.nodemanager.DeletionService; import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.ExitCode; import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.Signal; +import org.apache.hadoop.yarn.server.nodemanager.DeletionService; import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationState; import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer; import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceLocalizationService; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; -import org.apache.hadoop.yarn.server.security.ContainerTokenSecretManager; import org.apache.hadoop.yarn.util.ConverterUtils; import org.junit.Test; @@ -384,11 +383,9 @@ public class TestContainerManager extend delSrvc = new DeletionService(exec); delSrvc.init(conf); - ContainerTokenSecretManager containerTokenSecretManager = new - ContainerTokenSecretManager(conf); - containerManager = new ContainerManagerImpl(context, exec, delSrvc, - nodeStatusUpdater, metrics, containerTokenSecretManager, - new ApplicationACLsManager(conf), dirsHandler); + containerManager = + new ContainerManagerImpl(context, exec, delSrvc, nodeStatusUpdater, + metrics, new ApplicationACLsManager(conf), dirsHandler); containerManager.init(conf); containerManager.start(); Modified: hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java?rev=1398581&r1=1398580&r2=1398581&view=diff ============================================================================== --- hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java (original) +++ hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java Tue Oct 16 00:02:55 2012 @@ -30,9 +30,9 @@ import java.util.HashMap; import java.util.List; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.api.records.ApplicationAccessType; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; -import org.apache.hadoop.yarn.api.records.ApplicationAccessType; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.event.DrainDispatcher; @@ -50,8 +50,8 @@ import org.apache.hadoop.yarn.server.nod import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ApplicationLocalizationEvent; import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.LocalizationEvent; import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.LocalizationEventType; -import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerEventType; import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerEvent; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerEventType; import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitorEvent; import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitorEventType; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; Modified: hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResource.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResource.java?rev=1398581&r1=1398580&r2=1398581&view=diff ============================================================================== --- hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResource.java (original) +++ hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResource.java Tue Oct 16 00:02:55 2012 @@ -37,7 +37,7 @@ import static org.junit.Assert.*; public class TestLocalResource { static org.apache.hadoop.yarn.api.records.LocalResource getYarnResource(Path p, long size, - long timestamp, LocalResourceType type, LocalResourceVisibility state) + long timestamp, LocalResourceType type, LocalResourceVisibility state, String pattern) throws URISyntaxException { org.apache.hadoop.yarn.api.records.LocalResource ret = RecordFactoryProvider.getRecordFactory(null).newRecordInstance(org.apache.hadoop.yarn.api.records.LocalResource.class); ret.setResource(ConverterUtils.getYarnUrlFromURI(p.toUri())); @@ -45,6 +45,7 @@ public class TestLocalResource { ret.setTimestamp(timestamp); ret.setType(type); ret.setVisibility(state); + ret.setPattern(pattern); return ret; } @@ -72,9 +73,9 @@ public class TestLocalResource { long basetime = r.nextLong() >>> 2; org.apache.hadoop.yarn.api.records.LocalResource yA = getYarnResource( - new Path("http://yak.org:80/foobar"), -1, basetime, FILE, PUBLIC); + new Path("http://yak.org:80/foobar"), -1, basetime, FILE, PUBLIC, null); org.apache.hadoop.yarn.api.records.LocalResource yB = getYarnResource( - new Path("http://yak.org:80/foobar"), -1, basetime, FILE, PUBLIC); + new Path("http://yak.org:80/foobar"), -1, basetime, FILE, PUBLIC, null); final LocalResourceRequest a = new LocalResourceRequest(yA); LocalResourceRequest b = new LocalResourceRequest(yA); checkEqual(a, b); @@ -83,31 +84,37 @@ public class TestLocalResource { // ignore visibility yB = getYarnResource( - new Path("http://yak.org:80/foobar"), -1, basetime, FILE, PRIVATE); + new Path("http://yak.org:80/foobar"), -1, basetime, FILE, PRIVATE, null); b = new LocalResourceRequest(yB); checkEqual(a, b); // ignore size yB = getYarnResource( - new Path("http://yak.org:80/foobar"), 0, basetime, FILE, PRIVATE); + new Path("http://yak.org:80/foobar"), 0, basetime, FILE, PRIVATE, null); b = new LocalResourceRequest(yB); checkEqual(a, b); // note path yB = getYarnResource( - new Path("hdfs://dingo.org:80/foobar"), 0, basetime, ARCHIVE, PUBLIC); + new Path("hdfs://dingo.org:80/foobar"), 0, basetime, ARCHIVE, PUBLIC, null); b = new LocalResourceRequest(yB); checkNotEqual(a, b); // note type yB = getYarnResource( - new Path("http://yak.org:80/foobar"), 0, basetime, ARCHIVE, PUBLIC); + new Path("http://yak.org:80/foobar"), 0, basetime, ARCHIVE, PUBLIC, null); b = new LocalResourceRequest(yB); checkNotEqual(a, b); // note timestamp yB = getYarnResource( - new Path("http://yak.org:80/foobar"), 0, basetime + 1, FILE, PUBLIC); + new Path("http://yak.org:80/foobar"), 0, basetime + 1, FILE, PUBLIC, null); + b = new LocalResourceRequest(yB); + checkNotEqual(a, b); + + // note pattern + yB = getYarnResource( + new Path("http://yak.org:80/foobar"), 0, basetime + 1, FILE, PUBLIC, "^/foo/.*"); b = new LocalResourceRequest(yB); checkNotEqual(a, b); } @@ -120,24 +127,35 @@ public class TestLocalResource { System.out.println("SEED: " + seed); long basetime = r.nextLong() >>> 2; org.apache.hadoop.yarn.api.records.LocalResource yA = getYarnResource( - new Path("http://yak.org:80/foobar"), -1, basetime, FILE, PUBLIC); + new Path("http://yak.org:80/foobar"), -1, basetime, FILE, PUBLIC, "^/foo/.*"); final LocalResourceRequest a = new LocalResourceRequest(yA); // Path primary org.apache.hadoop.yarn.api.records.LocalResource yB = getYarnResource( - new Path("http://yak.org:80/foobaz"), -1, basetime, FILE, PUBLIC); + new Path("http://yak.org:80/foobaz"), -1, basetime, FILE, PUBLIC, "^/foo/.*"); LocalResourceRequest b = new LocalResourceRequest(yB); assertTrue(0 > a.compareTo(b)); // timestamp secondary yB = getYarnResource( - new Path("http://yak.org:80/foobar"), -1, basetime + 1, FILE, PUBLIC); + new Path("http://yak.org:80/foobar"), -1, basetime + 1, FILE, PUBLIC, "^/foo/.*"); b = new LocalResourceRequest(yB); assertTrue(0 > a.compareTo(b)); // type tertiary yB = getYarnResource( - new Path("http://yak.org:80/foobar"), -1, basetime, ARCHIVE, PUBLIC); + new Path("http://yak.org:80/foobar"), -1, basetime, ARCHIVE, PUBLIC, "^/foo/.*"); + b = new LocalResourceRequest(yB); + assertTrue(0 != a.compareTo(b)); // don't care about order, just ne + + // path 4th + yB = getYarnResource( + new Path("http://yak.org:80/foobar"), -1, basetime, ARCHIVE, PUBLIC, "^/food/.*"); + b = new LocalResourceRequest(yB); + assertTrue(0 != a.compareTo(b)); // don't care about order, just ne + + yB = getYarnResource( + new Path("http://yak.org:80/foobar"), -1, basetime, ARCHIVE, PUBLIC, null); b = new LocalResourceRequest(yB); assertTrue(0 != a.compareTo(b)); // don't care about order, just ne } Modified: hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResourcesTrackerImpl.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResourcesTrackerImpl.java?rev=1398581&r1=1398580&r2=1398581&view=diff ============================================================================== --- hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResourcesTrackerImpl.java (original) +++ hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResourcesTrackerImpl.java Tue Oct 16 00:02:55 2012 @@ -230,7 +230,7 @@ public class TestLocalResourcesTrackerIm long ts, LocalResourceVisibility vis) { final LocalResourceRequest req = new LocalResourceRequest(new Path("file:///tmp/" + user + "/rsrc" + i), - ts + i * 2000, LocalResourceType.FILE, vis); + ts + i * 2000, LocalResourceType.FILE, vis, null); return req; }