hadoop-yarn-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ss...@apache.org
Subject svn commit: r1398581 [4/9] - in /hadoop/common/branches/MR-3902/hadoop-yarn-project: ./ hadoop-yarn/ hadoop-yarn/bin/ hadoop-yarn/conf/ hadoop-yarn/hadoop-yarn-api/ hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ hadoop-yarn/hadoo...
Date Tue, 16 Oct 2012 00:04:13 GMT
Modified: hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java Tue Oct 16 00:02:55 2012
@@ -21,10 +21,10 @@ package org.apache.hadoop.yarn.server.no
 import static org.apache.hadoop.yarn.service.Service.STATE.STARTED;
 
 import java.io.IOException;
-import java.net.InetAddress;
 import java.net.InetSocketAddress;
 import java.nio.ByteBuffer;
 import java.util.Map;
+import java.util.Set;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -96,7 +96,6 @@ import org.apache.hadoop.yarn.server.nod
 import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
 import org.apache.hadoop.yarn.server.nodemanager.security.authorize.NMPolicyProvider;
 import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
-import org.apache.hadoop.yarn.server.security.ContainerTokenSecretManager;
 import org.apache.hadoop.yarn.service.CompositeService;
 import org.apache.hadoop.yarn.service.Service;
 import org.apache.hadoop.yarn.service.ServiceStateChangeListener;
@@ -110,14 +109,12 @@ public class ContainerManagerImpl extend
   final Context context;
   private final ContainersMonitor containersMonitor;
   private Server server;
-  private InetAddress resolvedAddress = null;
   private final ResourceLocalizationService rsrcLocalizationSrvc;
   private final ContainersLauncher containersLauncher;
   private final AuxServices auxiliaryServices;
   private final NodeManagerMetrics metrics;
 
   private final NodeStatusUpdater nodeStatusUpdater;
-  private ContainerTokenSecretManager containerTokenSecretManager;
 
   private final RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
 
@@ -129,8 +126,7 @@ public class ContainerManagerImpl extend
 
   public ContainerManagerImpl(Context context, ContainerExecutor exec,
       DeletionService deletionContext, NodeStatusUpdater nodeStatusUpdater,
-      NodeManagerMetrics metrics, ContainerTokenSecretManager 
-      containerTokenSecretManager, ApplicationACLsManager aclsManager,
+      NodeManagerMetrics metrics, ApplicationACLsManager aclsManager,
       LocalDirsHandlerService dirsHandler) {
     super(ContainerManagerImpl.class.getName());
     this.context = context;
@@ -149,7 +145,6 @@ public class ContainerManagerImpl extend
     addService(containersLauncher);
 
     this.nodeStatusUpdater = nodeStatusUpdater;
-    this.containerTokenSecretManager = containerTokenSecretManager;
     this.aclsManager = aclsManager;
 
     // Start configurable services
@@ -232,7 +227,7 @@ public class ContainerManagerImpl extend
 
     server =
         rpc.getServer(ContainerManager.class, this, initialAddress, conf,
-            this.containerTokenSecretManager,
+            this.context.getContainerTokenSecretManager(),
             conf.getInt(YarnConfiguration.NM_CONTAINER_MGR_THREAD_COUNT, 
                 YarnConfiguration.DEFAULT_NM_CONTAINER_MGR_THREAD_COUNT));
     
@@ -267,56 +262,77 @@ public class ContainerManagerImpl extend
     super.stop();
   }
 
+  // Get the remoteUGI corresponding to the api call.
+  private UserGroupInformation getRemoteUgi(String containerIDStr)
+      throws YarnRemoteException {
+    UserGroupInformation remoteUgi;
+    try {
+      remoteUgi = UserGroupInformation.getCurrentUser();
+    } catch (IOException e) {
+      String msg = "Cannot obtain the user-name for containerId: "
+          + containerIDStr + ". Got exception: "
+          + StringUtils.stringifyException(e);
+      LOG.warn(msg);
+      throw RPCUtil.getRemoteException(msg);
+    }
+    return remoteUgi;
+  }
+
+  // Obtain the needed ContainerTokenIdentifier from the remote-UGI. RPC layer
+  // currently sets only the required id, but iterate through anyways just to
+  // be sure.
+  private ContainerTokenIdentifier selectContainerTokenIdentifier(
+      UserGroupInformation remoteUgi) {
+    Set<TokenIdentifier> tokenIdentifiers = remoteUgi.getTokenIdentifiers();
+    ContainerTokenIdentifier resultId = null;
+    for (TokenIdentifier id : tokenIdentifiers) {
+      if (id instanceof ContainerTokenIdentifier) {
+        resultId = (ContainerTokenIdentifier) id;
+        break;
+      }
+    }
+    return resultId;
+  }
+
   /**
    * Authorize the request.
    * 
-   * @param containerID
+   * @param containerIDStr
    *          of the container
    * @param launchContext
    *          passed if verifying the startContainer, null otherwise.
+   * @param remoteUgi
+   *          ugi corresponding to the remote end making the api-call
    * @throws YarnRemoteException
    */
-  private void authorizeRequest(ContainerId containerID,
-      ContainerLaunchContext launchContext) throws YarnRemoteException {
+  private void authorizeRequest(String containerIDStr,
+      ContainerLaunchContext launchContext, UserGroupInformation remoteUgi)
+      throws YarnRemoteException {
 
     if (!UserGroupInformation.isSecurityEnabled()) {
       return;
     }
 
-    String containerIDStr = containerID.toString();
-
-    UserGroupInformation remoteUgi;
-    try {
-      remoteUgi = UserGroupInformation.getCurrentUser();
-    } catch (IOException e) {
-      String msg = "Cannot obtain the user-name for containerId: "
-          + containerIDStr + ". Got exception: "
-          + StringUtils.stringifyException(e);
-      LOG.warn(msg);
-      throw RPCUtil.getRemoteException(msg);
-    }
-
     boolean unauthorized = false;
-    StringBuilder messageBuilder = new StringBuilder(
-        "Unauthorized request to start container. ");
+    StringBuilder messageBuilder =
+        new StringBuilder("Unauthorized request to start container. ");
 
     if (!remoteUgi.getUserName().equals(containerIDStr)) {
       unauthorized = true;
       messageBuilder.append("\nExpected containerId: "
           + remoteUgi.getUserName() + " Found: " + containerIDStr);
-    }
-
-    if (launchContext != null) {
-
-      // Verify other things for startContainer() request.
+    } else if (launchContext != null) {
+      // Verify other things also for startContainer() request.
 
       if (LOG.isDebugEnabled()) {
-      LOG.debug("Number of TokenIdentifiers in the UGI from RPC: "
-          + remoteUgi.getTokenIdentifiers().size());
+        LOG.debug("Number of TokenIdentifiers in the UGI from RPC: "
+            + remoteUgi.getTokenIdentifiers().size());
       }
-      // We must and should get only one TokenIdentifier from the RPC.
-      ContainerTokenIdentifier tokenId = (ContainerTokenIdentifier) remoteUgi
-          .getTokenIdentifiers().iterator().next();
+
+      // Get the tokenId from the remote user ugi
+      ContainerTokenIdentifier tokenId =
+          selectContainerTokenIdentifier(remoteUgi);
+
       if (tokenId == null) {
         unauthorized = true;
         messageBuilder
@@ -324,6 +340,23 @@ public class ContainerManagerImpl extend
                 + containerIDStr);
       } else {
 
+        // Is the container coming in with correct user-name?
+        if (!tokenId.getApplicationSubmitter().equals(launchContext.getUser())) {
+          unauthorized = true;
+          messageBuilder.append("\n Expected user-name "
+              + tokenId.getApplicationSubmitter() + " but found "
+              + launchContext.getUser());
+        }
+
+        // Is the container being relaunched? Or RPC layer let startCall with 
+      	//  tokens generated off old-secret through?
+        if (!this.context.getContainerTokenSecretManager()
+          .isValidStartContainerRequest(tokenId)) {
+          unauthorized = true;
+          messageBuilder.append("\n Attempt to relaunch the same " +
+              "container with id " + containerIDStr + ".");
+        }
+
         // Ensure the token is not expired. 
         // Token expiry is not checked for stopContainer/getContainerStatus
         if (tokenId.getExpiryTimeStamp() < System.currentTimeMillis()) {
@@ -348,7 +381,7 @@ public class ContainerManagerImpl extend
       throw RPCUtil.getRemoteException(msg);
     }
   }
-
+  
   /**
    * Start a container on this NodeManager.
    */
@@ -359,10 +392,13 @@ public class ContainerManagerImpl extend
     ContainerLaunchContext launchContext = request.getContainerLaunchContext();
 
     ContainerId containerID = launchContext.getContainerId();
-    authorizeRequest(containerID, launchContext);
+    String containerIDStr = containerID.toString();
+
+    UserGroupInformation remoteUgi = getRemoteUgi(containerIDStr);
+    authorizeRequest(containerIDStr, launchContext, remoteUgi);
 
-    LOG.info("Start request for " + launchContext.getContainerId()
-        + " by user " + launchContext.getUser());
+    LOG.info("Start request for " + containerIDStr + " by user "
+        + launchContext.getUser());
 
     // //////////// Parse credentials
     ByteBuffer tokens = launchContext.getContainerTokens();
@@ -394,14 +430,14 @@ public class ContainerManagerImpl extend
           AuditConstants.START_CONTAINER, "ContainerManagerImpl",
           "Container already running on this node!",
           applicationID, containerID);
-      throw RPCUtil.getRemoteException("Container " + containerID
+      throw RPCUtil.getRemoteException("Container " + containerIDStr
           + " already is running on this node!!");
     }
 
     // Create the application
     Application application =
         new ApplicationImpl(dispatcher, this.aclsManager,
-            launchContext.getUser(), applicationID, credentials, context);
+          launchContext.getUser(), applicationID, credentials, context);
     if (null ==
         context.getApplications().putIfAbsent(applicationID, application)) {
       LOG.info("Creating a new application reference for app "
@@ -414,6 +450,12 @@ public class ContainerManagerImpl extend
     // TODO: Validate the request
     dispatcher.getEventHandler().handle(
         new ApplicationContainerInitEvent(container));
+    if (UserGroupInformation.isSecurityEnabled()) {
+      ContainerTokenIdentifier tokenId =
+          selectContainerTokenIdentifier(remoteUgi);
+      this.context.getContainerTokenSecretManager().startContainerSuccessful(
+        tokenId);
+    }
 
     NMAuditLogger.logSuccess(launchContext.getUser(), 
         AuditConstants.START_CONTAINER, "ContainerManageImpl", 
@@ -438,8 +480,12 @@ public class ContainerManagerImpl extend
       throws YarnRemoteException {
 
     ContainerId containerID = request.getContainerId();
+    String containerIDStr = containerID.toString();
+
     // TODO: Only the container's owner can kill containers today.
-    authorizeRequest(containerID, null);
+
+    UserGroupInformation remoteUgi = getRemoteUgi(containerIDStr);
+    authorizeRequest(containerIDStr, null, remoteUgi);
 
     StopContainerResponse response =
         recordFactory.newRecordInstance(StopContainerResponse.class);
@@ -476,10 +522,14 @@ public class ContainerManagerImpl extend
       GetContainerStatusRequest request) throws YarnRemoteException {
 
     ContainerId containerID = request.getContainerId();
+    String containerIDStr = containerID.toString();
+
     // TODO: Only the container's owner can get containers' status today.
-    authorizeRequest(containerID, null);
 
-    LOG.info("Getting container-status for " + containerID);
+    UserGroupInformation remoteUgi = getRemoteUgi(containerIDStr);
+    authorizeRequest(containerIDStr, null, remoteUgi);
+
+    LOG.info("Getting container-status for " + containerIDStr);
     Container container = this.context.getContainers().get(containerID);
     if (container != null) {
       ContainerStatus containerStatus = container.cloneAndGetContainerStatus();
@@ -490,7 +540,7 @@ public class ContainerManagerImpl extend
       return response;
     }
 
-    throw RPCUtil.getRemoteException("Container " + containerID
+    throw RPCUtil.getRemoteException("Container " + containerIDStr
         + " is not handled by this NodeManager");
   }
 

Modified: hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java Tue Oct 16 00:02:55 2012
@@ -28,8 +28,9 @@ import java.util.concurrent.locks.Reentr
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.security.Credentials;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.event.Dispatcher;
 import org.apache.hadoop.yarn.logaggregation.ContainerLogsRetentionPolicy;
@@ -42,6 +43,7 @@ import org.apache.hadoop.yarn.server.nod
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceLocalizationService;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ApplicationLocalizationEvent;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.LocalizationEventType;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.logaggregation.LogAggregationService;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerAppFinishedEvent;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerAppStartedEvent;
 import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
@@ -365,6 +367,10 @@ public class ApplicationImpl implements 
     @Override
     public void transition(ApplicationImpl app, ApplicationEvent event) {
 
+      // Inform the ContainerTokenSecretManager
+      if (UserGroupInformation.isSecurityEnabled()) {
+        app.context.getContainerTokenSecretManager().appFinished(app.appId);
+      }
       // Inform the logService
       app.dispatcher.getEventHandler().handle(
           new LogHandlerAppFinishedEvent(app.appId));

Modified: hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java Tue Oct 16 00:02:55 2012
@@ -209,6 +209,7 @@ public class ContainerLocalizer {
     }
     switch (rsrc.getType()) {
       case ARCHIVE:
+      case PATTERN:
         return 5 * rsrc.getSize();
       case FILE:
       default:

Modified: hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourceRequest.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourceRequest.java?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourceRequest.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourceRequest.java Tue Oct 16 00:02:55 2012
@@ -34,6 +34,7 @@ public class LocalResourceRequest
   private final long timestamp;
   private final LocalResourceType type;
   private final LocalResourceVisibility visibility;
+  private final String pattern;
 
   /**
    * Wrap API resource to match against cache of localized resources.
@@ -45,22 +46,28 @@ public class LocalResourceRequest
     this(ConverterUtils.getPathFromYarnURL(resource.getResource()),
         resource.getTimestamp(),
         resource.getType(),
-        resource.getVisibility());
+        resource.getVisibility(),
+        resource.getPattern());
   }
 
   LocalResourceRequest(Path loc, long timestamp, LocalResourceType type,
-      LocalResourceVisibility visibility) {
+      LocalResourceVisibility visibility, String pattern) {
     this.loc = loc;
     this.timestamp = timestamp;
     this.type = type;
     this.visibility = visibility;
+    this.pattern = pattern;
   }
 
   @Override
   public int hashCode() {
-    return loc.hashCode() ^
+    int hash = loc.hashCode() ^
       (int)((timestamp >>> 32) ^ timestamp) *
       type.hashCode();
+    if(pattern != null) {
+      hash = hash ^ pattern.hashCode();
+    }
+    return hash;
   }
 
   @Override
@@ -72,9 +79,14 @@ public class LocalResourceRequest
       return false;
     }
     final LocalResourceRequest other = (LocalResourceRequest) o;
+    String pattern = getPattern();
+    String otherPattern = other.getPattern();
+    boolean patternEquals = (pattern == null && otherPattern == null) || 
+       (pattern != null && otherPattern != null && pattern.equals(otherPattern)); 
     return getPath().equals(other.getPath()) &&
            getTimestamp() == other.getTimestamp() &&
-           getType() == other.getType();
+           getType() == other.getType() &&
+           patternEquals;
   }
 
   @Override
@@ -87,6 +99,19 @@ public class LocalResourceRequest
       ret = (int)(getTimestamp() - other.getTimestamp());
       if (0 == ret) {
         ret = getType().ordinal() - other.getType().ordinal();
+        if (0 == ret) {
+          String pattern = getPattern();
+          String otherPattern = other.getPattern();
+          if (pattern == null && otherPattern == null) {
+            ret = 0;
+          } else if (pattern == null) {
+            ret = -1;
+          } else if (otherPattern == null) {
+            ret = 1;
+          } else {
+            ret = pattern.compareTo(otherPattern);    
+          }
+        }
       }
     }
     return ret;
@@ -122,6 +147,11 @@ public class LocalResourceRequest
   }
 
   @Override
+  public String getPattern() {
+    return pattern;
+  }
+  
+  @Override
   public void setResource(URL resource) {
     throw new UnsupportedOperationException();
   }
@@ -145,14 +175,20 @@ public class LocalResourceRequest
   public void setVisibility(LocalResourceVisibility visibility) {
     throw new UnsupportedOperationException();
   }
-
+  
+  @Override
+  public void setPattern(String pattern) {
+    throw new UnsupportedOperationException();
+  }
+  
   @Override
   public String toString() {
     StringBuilder sb = new StringBuilder();
     sb.append("{ ");
     sb.append(getPath().toString()).append(", ");
     sb.append(getTimestamp()).append(", ");
-    sb.append(getType()).append(" }");
+    sb.append(getType()).append(", ");
+    sb.append(getPattern()).append(" }");
     return sb.toString();
   }
 }

Modified: hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalizedResource.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalizedResource.java?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalizedResource.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalizedResource.java Tue Oct 16 00:02:55 2012
@@ -219,7 +219,8 @@ public class LocalizedResource implement
       ContainerId container = ctxt.getContainerId();
       rsrc.ref.add(container);
       rsrc.dispatcher.getEventHandler().handle(
-          new LocalizerResourceRequestEvent(rsrc, req.getVisibility(), ctxt));
+          new LocalizerResourceRequestEvent(rsrc, req.getVisibility(), ctxt, 
+              req.getLocalResourceRequest().getPattern()));
     }
   }
 

Modified: hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java Tue Oct 16 00:02:55 2012
@@ -181,6 +181,7 @@ public class ResourceLocalizationService
     try {
       // TODO queue deletions here, rather than NM init?
       FileContext lfs = getLocalFileContext(conf);
+      lfs.setUMask(new FsPermission((short)FsPermission.DEFAULT_UMASK));
       List<String> localDirs = dirsHandler.getLocalDirs();
       for (String localDir : localDirs) {
         // $local/usercache
@@ -751,6 +752,7 @@ public class ResourceLocalizationService
           next.setTimestamp(nextRsrc.getTimestamp());
           next.setType(nextRsrc.getType());
           next.setVisibility(evt.getVisibility());
+          next.setPattern(evt.getPattern());
           scheduled.put(nextRsrc, evt);
           return next;
         }

Modified: hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/event/LocalizerResourceRequestEvent.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/event/LocalizerResourceRequestEvent.java?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/event/LocalizerResourceRequestEvent.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/event/LocalizerResourceRequestEvent.java Tue Oct 16 00:02:55 2012
@@ -32,14 +32,16 @@ public class LocalizerResourceRequestEve
   private final LocalizerContext context;
   private final LocalizedResource resource;
   private final LocalResourceVisibility vis;
+  private final String pattern;
 
   public LocalizerResourceRequestEvent(LocalizedResource resource,
-      LocalResourceVisibility vis, LocalizerContext context) {
+      LocalResourceVisibility vis, LocalizerContext context, String pattern) {
     super(LocalizerEventType.REQUEST_RESOURCE_LOCALIZATION,
         ConverterUtils.toString(context.getContainerId()));
     this.vis = vis;
     this.context = context;
     this.resource = resource;
+    this.pattern = pattern;
   }
 
   public LocalizedResource getResource() {
@@ -54,4 +56,8 @@ public class LocalizerResourceRequestEve
     return vis;
   }
 
+  public String getPattern() {
+    return pattern;
+  }
+
 }

Modified: hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregator.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregator.java?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregator.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregator.java Tue Oct 16 00:02:55 2012
@@ -26,7 +26,4 @@ public interface AppLogAggregator extend
       boolean wasContainerSuccessful);
 
   void finishLogAggregation();
-
-  void join();
-
 }

Modified: hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java Tue Oct 16 00:02:55 2012
@@ -137,6 +137,9 @@ public class AppLogAggregatorImpl implem
     try {
       doAppLogAggregation();
     } finally {
+      if (!this.appAggregationFinished.get()) {
+        LOG.warn("Aggregation did not complete for application " + appId);
+      }
       this.appAggregationFinished.set(true);
     }
   }
@@ -155,6 +158,7 @@ public class AppLogAggregatorImpl implem
         }
       } catch (InterruptedException e) {
         LOG.warn("PendingContainers queue is interrupted");
+        this.appFinishing.set(true);
       }
     }
 
@@ -197,6 +201,7 @@ public class AppLogAggregatorImpl implem
     this.dispatcher.getEventHandler().handle(
         new ApplicationEvent(this.appId,
             ApplicationEventType.APPLICATION_LOG_HANDLING_FINISHED));
+    this.appAggregationFinished.set(true);    
   }
 
   private Path getRemoteNodeTmpLogFileForApp() {
@@ -250,21 +255,4 @@ public class AppLogAggregatorImpl implem
     LOG.info("Application just finished : " + this.applicationId);
     this.appFinishing.set(true);
   }
-
-  @Override
-  public void join() {
-    // Aggregation service is finishing
-    this.finishLogAggregation();
-
-    while (!this.appAggregationFinished.get()) {
-      LOG.info("Waiting for aggregation to complete for "
-          + this.applicationId);
-      try {
-        Thread.sleep(THREAD_SLEEP_TIME);
-      } catch (InterruptedException e) {
-        LOG.warn("Join interrupted. Some logs may not have been aggregated!!");
-        break;
-      }
-    }
-  }
 }

Modified: hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java Tue Oct 16 00:02:55 2012
@@ -25,6 +25,7 @@ import java.util.concurrent.ConcurrentHa
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -35,8 +36,6 @@ import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.security.Credentials;
 import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.security.token.TokenIdentifier;
 import org.apache.hadoop.yarn.YarnException;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
@@ -137,11 +136,33 @@ public class LogAggregationService exten
   @Override
   public synchronized void stop() {
     LOG.info(this.getName() + " waiting for pending aggregation during exit");
-    for (AppLogAggregator appLogAggregator : this.appLogAggregators.values()) {
-      appLogAggregator.join();
-    }
+    stopAggregators();
     super.stop();
   }
+   
+  private void stopAggregators() {
+    threadPool.shutdown();
+    // politely ask to finish
+    for (AppLogAggregator aggregator : appLogAggregators.values()) {
+      aggregator.finishLogAggregation();
+    }
+    while (!threadPool.isTerminated()) { // wait for all threads to finish
+      for (ApplicationId appId : appLogAggregators.keySet()) {
+        LOG.info("Waiting for aggregation to complete for " + appId);
+      }
+      try {
+        if (!threadPool.awaitTermination(30, TimeUnit.SECONDS)) {
+          threadPool.shutdownNow(); // send interrupt to hurry them along
+        }
+      } catch (InterruptedException e) {
+        LOG.warn("Aggregation stop interrupted!");
+        break;
+      }
+    }
+    for (ApplicationId appId : appLogAggregators.keySet()) {
+      LOG.warn("Some logs may not have been aggregated for " + appId);
+    }
+  }
   
   private void verifyAndCreateRemoteLogDir(Configuration conf) {
     // Checking the existance of the TLD
@@ -293,10 +314,7 @@ public class LogAggregationService exten
     final UserGroupInformation userUgi =
         UserGroupInformation.createRemoteUser(user);
     if (credentials != null) {
-      for (Token<? extends TokenIdentifier> token : credentials
-          .getAllTokens()) {
-        userUgi.addToken(token);
-      }
+      userUgi.addCredentials(credentials);
     }
 
     // New application
@@ -312,9 +330,13 @@ public class LogAggregationService exten
     try {
       // Create the app dir
       createAppDir(user, appId, userUgi);
-    } catch (YarnException e) {
+    } catch (Exception e) {
+      appLogAggregators.remove(appId);
       closeFileSystems(userUgi);
-      throw e;
+      if (!(e instanceof YarnException)) {
+        e = new YarnException(e);
+      }
+      throw (YarnException)e;
     }
 
 

Modified: hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/NonAggregatingLogHandler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/NonAggregatingLogHandler.java?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/NonAggregatingLogHandler.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/NonAggregatingLogHandler.java Tue Oct 16 00:02:55 2012
@@ -79,16 +79,18 @@ public class NonAggregatingLogHandler ex
 
   @Override
   public void stop() {
-    sched.shutdown();
-    boolean isShutdown = false;
-    try {
-      isShutdown = sched.awaitTermination(10, TimeUnit.SECONDS);
-    } catch (InterruptedException e) {
-      sched.shutdownNow();
-      isShutdown = true;
-    }
-    if (!isShutdown) {
-      sched.shutdownNow();
+    if (sched != null) {
+      sched.shutdown();
+      boolean isShutdown = false;
+      try {
+        isShutdown = sched.awaitTermination(10, TimeUnit.SECONDS);
+      } catch (InterruptedException e) {
+        sched.shutdownNow();
+        isShutdown = true;
+      }
+      if (!isShutdown) {
+        sched.shutdownNow();
+      }
     }
     super.stop();
   }

Modified: hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java Tue Oct 16 00:02:55 2012
@@ -37,7 +37,7 @@ import org.apache.hadoop.yarn.server.nod
 import org.apache.hadoop.yarn.server.nodemanager.Context;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerKillEvent;
 import org.apache.hadoop.yarn.service.AbstractService;
-import org.apache.hadoop.yarn.util.ProcfsBasedProcessTree;
+import org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree;
 import org.apache.hadoop.yarn.util.ResourceCalculatorPlugin;
 
 import com.google.common.base.Preconditions;
@@ -60,6 +60,8 @@ public class ContainersMonitorImpl exten
   private final Dispatcher eventDispatcher;
   private final Context context;
   private ResourceCalculatorPlugin resourceCalculatorPlugin;
+  private Configuration conf;
+  private Class<? extends ResourceCalculatorProcessTree> processTreeClass;
 
   private long maxVmemAllottedForContainers = DISABLED_MEMORY_LIMIT;
   private long maxPmemAllottedForContainers = DISABLED_MEMORY_LIMIT;
@@ -96,6 +98,11 @@ public class ContainersMonitorImpl exten
         ResourceCalculatorPlugin.getResourceCalculatorPlugin(clazz, conf);
     LOG.info(" Using ResourceCalculatorPlugin : "
         + this.resourceCalculatorPlugin);
+    processTreeClass = conf.getClass(YarnConfiguration.NM_CONTAINER_MON_PROCESS_TREE, null,
+            ResourceCalculatorProcessTree.class);
+    this.conf = conf;
+    LOG.info(" Using ResourceCalculatorProcessTree : "
+        + this.processTreeClass);
 
     long totalPhysicalMemoryOnNM = DISABLED_MEMORY_LIMIT;
     if (this.resourceCalculatorPlugin != null) {
@@ -140,7 +147,7 @@ public class ContainersMonitorImpl exten
 
   /**
    * Is the total physical memory check enabled?
-   * 
+   *
    * @return true if total physical memory check is enabled.
    */
   boolean isPhysicalMemoryCheckEnabled() {
@@ -149,7 +156,7 @@ public class ContainersMonitorImpl exten
 
   /**
    * Is the total virtual memory check enabled?
-   * 
+   *
    * @return true if total virtual memory check is enabled.
    */
   boolean isVirtualMemoryCheckEnabled() {
@@ -157,12 +164,16 @@ public class ContainersMonitorImpl exten
   }
 
   private boolean isEnabled() {
-    if (!ProcfsBasedProcessTree.isAvailable()) {
-      LOG.info("ProcessTree implementation is missing on this system. "
-          + this.getClass().getName() + " is disabled.");
-      return false;
+    if (resourceCalculatorPlugin == null) {
+            LOG.info("ResourceCalculatorPlugin is unavailable on this system. "
+                + this.getClass().getName() + " is disabled.");
+            return false;
+    }
+    if (ResourceCalculatorProcessTree.getResourceCalculatorProcessTree("0", processTreeClass, conf) == null) {
+        LOG.info("ResourceCalculatorProcessTree is unavailable on this system. "
+                + this.getClass().getName() + " is disabled.");
+            return false;
     }
-
     if (!(isPhysicalMemoryCheckEnabled() || isVirtualMemoryCheckEnabled())) {
       LOG.info("Neither virutal-memory nor physical-memory monitoring is " +
           "needed. Not running the monitor-thread");
@@ -196,12 +207,12 @@ public class ContainersMonitorImpl exten
   private static class ProcessTreeInfo {
     private ContainerId containerId;
     private String pid;
-    private ProcfsBasedProcessTree pTree;
+    private ResourceCalculatorProcessTree pTree;
     private long vmemLimit;
     private long pmemLimit;
 
     public ProcessTreeInfo(ContainerId containerId, String pid,
-        ProcfsBasedProcessTree pTree, long vmemLimit, long pmemLimit) {
+        ResourceCalculatorProcessTree pTree, long vmemLimit, long pmemLimit) {
       this.containerId = containerId;
       this.pid = pid;
       this.pTree = pTree;
@@ -221,11 +232,11 @@ public class ContainersMonitorImpl exten
       this.pid = pid;
     }
 
-    public ProcfsBasedProcessTree getProcessTree() {
+    public ResourceCalculatorProcessTree getProcessTree() {
       return this.pTree;
     }
 
-    public void setProcessTree(ProcfsBasedProcessTree pTree) {
+    public void setProcessTree(ResourceCalculatorProcessTree pTree) {
       this.pTree = pTree;
     }
 
@@ -245,20 +256,20 @@ public class ContainersMonitorImpl exten
   /**
    * Check whether a container's process tree's current memory usage is over
    * limit.
-   * 
+   *
    * When a java process exec's a program, it could momentarily account for
    * double the size of it's memory, because the JVM does a fork()+exec()
    * which at fork time creates a copy of the parent's memory. If the
    * monitoring thread detects the memory used by the container tree at the
    * same instance, it could assume it is over limit and kill the tree, for no
    * fault of the process itself.
-   * 
+   *
    * We counter this problem by employing a heuristic check: - if a process
    * tree exceeds the memory limit by more than twice, it is killed
    * immediately - if a process tree has processes older than the monitoring
    * interval exceeding the memory limit by even 1 time, it is killed. Else it
    * is given the benefit of doubt to lie around for one more iteration.
-   * 
+   *
    * @param containerId
    *          Container Id for the container tree
    * @param currentMemUsage
@@ -295,7 +306,7 @@ public class ContainersMonitorImpl exten
   }
 
   // method provided just for easy testing purposes
-  boolean isProcessTreeOverLimit(ProcfsBasedProcessTree pTree,
+  boolean isProcessTreeOverLimit(ResourceCalculatorProcessTree pTree,
       String containerId, long limit) {
     long currentMemUsage = pTree.getCumulativeVmem();
     // as processes begin with an age 1, we want to see if there are processes
@@ -370,9 +381,8 @@ public class ContainersMonitorImpl exten
                 LOG.debug("Tracking ProcessTree " + pId
                     + " for the first time");
 
-                ProcfsBasedProcessTree pt =
-                    new ProcfsBasedProcessTree(pId,
-                        ContainerExecutor.isSetsidAvailable);
+                ResourceCalculatorProcessTree pt =
+                    ResourceCalculatorProcessTree.getResourceCalculatorProcessTree(pId, processTreeClass, conf);
                 ptInfo.setPid(pId);
                 ptInfo.setProcessTree(pt);
               }
@@ -385,7 +395,7 @@ public class ContainersMonitorImpl exten
 
             LOG.debug("Constructing ProcessTree for : PID = " + pId
                 + " ContainerId = " + containerId);
-            ProcfsBasedProcessTree pTree = ptInfo.getProcessTree();
+            ResourceCalculatorProcessTree pTree = ptInfo.getProcessTree();
             pTree = pTree.getProcessTree(); // get the updated process-tree
             ptInfo.setProcessTree(pTree); // update ptInfo with proces-tree of
                                           // updated state
@@ -471,7 +481,7 @@ public class ContainersMonitorImpl exten
     private String formatErrorMessage(String memTypeExceeded,
         long currentVmemUsage, long vmemLimit,
         long currentPmemUsage, long pmemLimit,
-        String pId, ContainerId containerId, ProcfsBasedProcessTree pTree) {
+        String pId, ContainerId containerId, ResourceCalculatorProcessTree pTree) {
       return
         String.format("Container [pid=%s,containerID=%s] is running beyond %s memory limits. ",
             pId, containerId, memTypeExceeded) +

Modified: hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java Tue Oct 16 00:02:55 2012
@@ -27,8 +27,6 @@ import org.apache.commons.logging.LogFac
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
-import org.apache.hadoop.yarn.server.security.ContainerTokenSecretManager;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManagerImpl;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEvent;
@@ -50,6 +48,7 @@ import org.apache.hadoop.yarn.server.nod
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.LogHandler;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerEvent;
 import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
+import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
 
 public class DummyContainerManager extends ContainerManagerImpl {
 
@@ -59,11 +58,10 @@ public class DummyContainerManager exten
   public DummyContainerManager(Context context, ContainerExecutor exec,
       DeletionService deletionContext, NodeStatusUpdater nodeStatusUpdater,
       NodeManagerMetrics metrics,
-      ContainerTokenSecretManager containerTokenSecretManager,
       ApplicationACLsManager applicationACLsManager,
       LocalDirsHandlerService dirsHandler) {
     super(context, exec, deletionContext, nodeStatusUpdater, metrics,
-        containerTokenSecretManager, applicationACLsManager, dirsHandler);
+      applicationACLsManager, dirsHandler);
   }
 
   @Override

Modified: hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDefaultContainerExecutor.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDefaultContainerExecutor.java?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDefaultContainerExecutor.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDefaultContainerExecutor.java Tue Oct 16 00:02:55 2012
@@ -18,6 +18,7 @@
 
 package org.apache.hadoop.yarn.server.nodemanager;
 
+import java.io.FileNotFoundException;
 import java.io.InputStream;
 import java.io.IOException;
 import java.net.InetSocketAddress;
@@ -26,8 +27,11 @@ import java.util.EnumSet;
 import java.util.List;
 import java.util.Random;
 
+import junit.framework.Assert;
+
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.AbstractFileSystem;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileContext;
@@ -38,6 +42,7 @@ import org.apache.hadoop.fs.permission.F
 import org.apache.hadoop.io.DataInputBuffer;
 import org.apache.hadoop.io.DataOutputBuffer;
 import org.apache.hadoop.util.Progressable;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.FakeFSDataInputStream;
 
 import static org.apache.hadoop.fs.CreateFlag.*;
@@ -91,11 +96,16 @@ public class TestDefaultContainerExecuto
   }
   */
 
+  private static final Path BASE_TMP_PATH = new Path("target",
+      TestDefaultContainerExecutor.class.getSimpleName());
+
   @AfterClass
   public static void deleteTmpFiles() throws IOException {
     FileContext lfs = FileContext.getLocalFSFileContext();
-    lfs.delete(new Path("target",
-          TestDefaultContainerExecutor.class.getSimpleName()), true);
+    try {
+      lfs.delete(BASE_TMP_PATH, true);
+    } catch (FileNotFoundException e) {
+    }
   }
 
   byte[] createTmpFile(Path dst, Random r, int len)
@@ -116,6 +126,71 @@ public class TestDefaultContainerExecuto
     return bytes;
   }
 
+  @Test
+  public void testDirPermissions() throws Exception {
+    deleteTmpFiles();
+
+    final String user = "somebody";
+    final String appId = "app_12345_123";
+    final FsPermission userCachePerm = new FsPermission(
+        DefaultContainerExecutor.USER_PERM);
+    final FsPermission appCachePerm = new FsPermission(
+        DefaultContainerExecutor.APPCACHE_PERM);
+    final FsPermission fileCachePerm = new FsPermission(
+        DefaultContainerExecutor.FILECACHE_PERM);
+    final FsPermission appDirPerm = new FsPermission(
+        DefaultContainerExecutor.APPDIR_PERM);
+    final FsPermission logDirPerm = new FsPermission(
+        DefaultContainerExecutor.LOGDIR_PERM);
+    List<String> localDirs = new ArrayList<String>();
+    localDirs.add(new Path(BASE_TMP_PATH, "localDirA").toString());
+    localDirs.add(new Path(BASE_TMP_PATH, "localDirB").toString());
+    List<String> logDirs = new ArrayList<String>();
+    logDirs.add(new Path(BASE_TMP_PATH, "logDirA").toString());
+    logDirs.add(new Path(BASE_TMP_PATH, "logDirB").toString());
+
+    Configuration conf = new Configuration();
+    conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "077");
+    FileContext lfs = FileContext.getLocalFSFileContext(conf);
+    DefaultContainerExecutor executor = new DefaultContainerExecutor(lfs);
+    executor.init();
+
+    try {
+      executor.createUserLocalDirs(localDirs, user);
+      executor.createUserCacheDirs(localDirs, user);
+      executor.createAppDirs(localDirs, user, appId);
+
+      for (String dir : localDirs) {
+        FileStatus stats = lfs.getFileStatus(
+            new Path(new Path(dir, ContainerLocalizer.USERCACHE), user));
+        Assert.assertEquals(userCachePerm, stats.getPermission());
+      }
+
+      for (String dir : localDirs) {
+        Path userCachePath = new Path(
+            new Path(dir, ContainerLocalizer.USERCACHE), user);
+        Path appCachePath = new Path(userCachePath,
+            ContainerLocalizer.APPCACHE);
+        FileStatus stats = lfs.getFileStatus(appCachePath);
+        Assert.assertEquals(appCachePerm, stats.getPermission());
+        stats = lfs.getFileStatus(
+            new Path(userCachePath, ContainerLocalizer.FILECACHE));
+        Assert.assertEquals(fileCachePerm, stats.getPermission());
+        stats = lfs.getFileStatus(new Path(appCachePath, appId));
+        Assert.assertEquals(appDirPerm, stats.getPermission());
+      }
+
+      executor.createAppLogDirs(appId, logDirs);
+
+      for (String dir : logDirs) {
+        FileStatus stats = lfs.getFileStatus(new Path(dir, appId));
+        Assert.assertEquals(logDirPerm, stats.getPermission());
+      }
+    } finally {
+      deleteTmpFiles();
+    }
+  }
+
 //  @Test
 //  public void testInit() throws IOException, InterruptedException {
 //    Configuration conf = new Configuration();

Modified: hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDirectoryCollection.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDirectoryCollection.java?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDirectoryCollection.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDirectoryCollection.java Tue Oct 16 00:02:55 2012
@@ -23,7 +23,13 @@ import java.io.IOException;
 import java.util.List;
 import java.util.ListIterator;
 
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.fs.FileContext;
+import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
 import org.junit.AfterClass;
 import org.junit.Assert;
 import org.junit.BeforeClass;
@@ -65,4 +71,37 @@ public class TestDirectoryCollection {
     // Verify no ConcurrentModification is thrown
     li.next();
   }
+
+  @Test
+  public void testCreateDirectories() throws IOException {
+    Configuration conf = new Configuration();
+    conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "077");
+    FileContext localFs = FileContext.getLocalFSFileContext(conf);
+
+    String dirA = new File(testDir, "dirA").getPath();
+    String dirB = new File(dirA, "dirB").getPath();
+    String dirC = new File(testDir, "dirC").getPath();
+    Path pathC = new Path(dirC);
+    FsPermission permDirC = new FsPermission((short)0710);
+
+    localFs.mkdir(pathC, null, true);
+    localFs.setPermission(pathC, permDirC);
+
+    String[] dirs = { dirA, dirB, dirC };
+    DirectoryCollection dc = new DirectoryCollection(dirs);
+    FsPermission defaultPerm = FsPermission.getDefault()
+        .applyUMask(new FsPermission((short)FsPermission.DEFAULT_UMASK));
+    boolean createResult = dc.createNonExistentDirs(localFs, defaultPerm);
+    Assert.assertTrue(createResult);
+
+    FileStatus status = localFs.getFileStatus(new Path(dirA));
+    Assert.assertEquals("local dir parent not created with proper permissions",
+        defaultPerm, status.getPermission());
+    status = localFs.getFileStatus(new Path(dirB));
+    Assert.assertEquals("local dir not created with proper permissions",
+        defaultPerm, status.getPermission());
+    status = localFs.getFileStatus(pathC);
+    Assert.assertEquals("existing local directory permissions modified",
+        permDirC, status.getPermission());
+  }
 }

Modified: hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java Tue Oct 16 00:02:55 2012
@@ -36,12 +36,12 @@ import org.apache.hadoop.yarn.event.Asyn
 import org.apache.hadoop.yarn.event.Dispatcher;
 import org.apache.hadoop.yarn.factories.RecordFactory;
 import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
-import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
-import org.apache.hadoop.yarn.server.security.ContainerTokenSecretManager;
 import org.apache.hadoop.yarn.server.api.ResourceTracker;
 import org.apache.hadoop.yarn.server.nodemanager.NodeManager.NMContext;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.BaseContainerManagerTest;
 import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
+import org.apache.hadoop.yarn.server.nodemanager.security.NMContainerTokenSecretManager;
+import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
 import org.junit.Test;
 
 public class TestEventFlow {
@@ -69,9 +69,9 @@ public class TestEventFlow {
     localLogDir.mkdir();
     remoteLogDir.mkdir();
 
-    Context context = new NMContext();
-
     YarnConfiguration conf = new YarnConfiguration();
+    Context context = new NMContext(new NMContainerTokenSecretManager(conf));
+
     conf.set(YarnConfiguration.NM_LOCAL_DIRS, localDir.getAbsolutePath());
     conf.set(YarnConfiguration.NM_LOG_DIRS, localLogDir.getAbsolutePath());
     conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, 
@@ -86,10 +86,8 @@ public class TestEventFlow {
     healthChecker.init(conf);
     LocalDirsHandlerService dirsHandler = healthChecker.getDiskHandler();
     NodeManagerMetrics metrics = NodeManagerMetrics.create();
-    ContainerTokenSecretManager containerTokenSecretManager =
-        new ContainerTokenSecretManager(conf);
     NodeStatusUpdater nodeStatusUpdater =
-        new NodeStatusUpdaterImpl(context, dispatcher, healthChecker, metrics, containerTokenSecretManager) {
+        new NodeStatusUpdaterImpl(context, dispatcher, healthChecker, metrics) {
       @Override
       protected ResourceTracker getRMClient() {
         return new LocalRMInterface();
@@ -101,10 +99,9 @@ public class TestEventFlow {
       }
     };
 
-    DummyContainerManager containerManager = new DummyContainerManager(
-        context, exec, del, nodeStatusUpdater, metrics,
-        containerTokenSecretManager, new ApplicationACLsManager(conf),
-        dirsHandler);
+    DummyContainerManager containerManager =
+        new DummyContainerManager(context, exec, del, nodeStatusUpdater,
+          metrics, new ApplicationACLsManager(conf), dirsHandler);
     containerManager.init(conf);
     containerManager.start();
 

Modified: hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNMAuditLogger.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNMAuditLogger.java?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNMAuditLogger.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNMAuditLogger.java Tue Oct 16 00:02:55 2012
@@ -17,37 +17,29 @@
  */
 package org.apache.hadoop.yarn.server.nodemanager;
 
+import static junit.framework.Assert.assertEquals;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
 import java.net.InetAddress;
 import java.net.InetSocketAddress;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.ipc.RPC;
 import org.apache.hadoop.ipc.Server;
 import org.apache.hadoop.ipc.TestRPC.TestImpl;
 import org.apache.hadoop.ipc.TestRPC.TestProtocol;
+import org.apache.hadoop.net.NetUtils;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.server.nodemanager.NMAuditLogger;
-import org.apache.hadoop.yarn.server.nodemanager.NMAuditLogger.AuditConstants;
 import org.apache.hadoop.yarn.server.nodemanager.NMAuditLogger.Keys;
-import org.apache.hadoop.yarn.server.security.ContainerTokenSecretManager;
-
-import org.apache.hadoop.net.NetUtils;
-
-import static org.mockito.Mockito.*;
-import static junit.framework.Assert.*;
-import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
 
-
 /**
  * Tests {@link NMAuditLogger}.
  */
 public class TestNMAuditLogger {
-  private static final Log LOG = LogFactory.getLog(TestNMAuditLogger.class);
   private static final String USER = "test";
   private static final String OPERATION = "oper";
   private static final String TARGET = "tgt";
@@ -212,8 +204,10 @@ public class TestNMAuditLogger {
   public void testNMAuditLoggerWithIP() throws Exception {
     Configuration conf = new Configuration();
     // start the IPC server
-    Server server = RPC.getServer(TestProtocol.class,
-        new MyTestRPCServer(), "0.0.0.0", 0, 5, true, conf, null);
+    Server server = new RPC.Builder(conf).setProtocol(TestProtocol.class)
+        .setInstance(new MyTestRPCServer()).setBindAddress("0.0.0.0")
+        .setPort(0).setNumHandlers(5).setVerbose(true).build();
+
     server.start();
 
     InetSocketAddress addr = NetUtils.getConnectAddress(server);

Modified: hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java Tue Oct 16 00:02:55 2012
@@ -65,7 +65,6 @@ import org.apache.hadoop.yarn.server.nod
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerImpl;
 import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
 import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
-import org.apache.hadoop.yarn.server.security.ContainerTokenSecretManager;
 import org.apache.hadoop.yarn.service.Service;
 import org.apache.hadoop.yarn.service.Service.STATE;
 import org.apache.hadoop.yarn.util.BuilderUtils;
@@ -234,10 +233,8 @@ public class TestNodeStatusUpdater {
     private Context context;
 
     public MyNodeStatusUpdater(Context context, Dispatcher dispatcher,
-        NodeHealthCheckerService healthChecker, NodeManagerMetrics metrics,
-        ContainerTokenSecretManager containerTokenSecretManager) {
-      super(context, dispatcher, healthChecker, metrics,
-          containerTokenSecretManager);
+        NodeHealthCheckerService healthChecker, NodeManagerMetrics metrics) {
+      super(context, dispatcher, healthChecker, metrics);
       this.context = context;
     }
 
@@ -252,10 +249,8 @@ public class TestNodeStatusUpdater {
     private Context context;
 
     public MyNodeStatusUpdater3(Context context, Dispatcher dispatcher,
-        NodeHealthCheckerService healthChecker, NodeManagerMetrics metrics,
-        ContainerTokenSecretManager containerTokenSecretManager) {
-      super(context, dispatcher, healthChecker, metrics,
-          containerTokenSecretManager);
+        NodeHealthCheckerService healthChecker, NodeManagerMetrics metrics) {
+      super(context, dispatcher, healthChecker, metrics);
       this.context = context;
       this.resourceTracker = new MyResourceTracker3(this.context);
     }
@@ -266,7 +261,7 @@ public class TestNodeStatusUpdater {
     }
     
     @Override
-    protected boolean isSecurityEnabled() {
+    protected boolean isTokenKeepAliveEnabled(Configuration conf) {
       return true;
     }
   }
@@ -276,11 +271,9 @@ public class TestNodeStatusUpdater {
     private MyNodeStatusUpdater3 nodeStatusUpdater;
     @Override
     protected NodeStatusUpdater createNodeStatusUpdater(Context context,
-        Dispatcher dispatcher, NodeHealthCheckerService healthChecker,
-        ContainerTokenSecretManager containerTokenSecretManager) {
+        Dispatcher dispatcher, NodeHealthCheckerService healthChecker) {
       this.nodeStatusUpdater =
-          new MyNodeStatusUpdater3(context, dispatcher, healthChecker, metrics,
-              containerTokenSecretManager);
+          new MyNodeStatusUpdater3(context, dispatcher, healthChecker, metrics);
       return this.nodeStatusUpdater;
     }
 
@@ -398,10 +391,9 @@ public class TestNodeStatusUpdater {
     nm = new NodeManager() {
       @Override
       protected NodeStatusUpdater createNodeStatusUpdater(Context context,
-          Dispatcher dispatcher, NodeHealthCheckerService healthChecker,
-          ContainerTokenSecretManager containerTokenSecretManager) {
+          Dispatcher dispatcher, NodeHealthCheckerService healthChecker) {
         return new MyNodeStatusUpdater(context, dispatcher, healthChecker,
-                                       metrics, containerTokenSecretManager);
+                                       metrics);
       }
     };
 
@@ -528,11 +520,9 @@ public class TestNodeStatusUpdater {
     nm = new NodeManager() {
       @Override
       protected NodeStatusUpdater createNodeStatusUpdater(Context context,
-          Dispatcher dispatcher, NodeHealthCheckerService healthChecker,
-          ContainerTokenSecretManager containerTokenSecretManager) {
+          Dispatcher dispatcher, NodeHealthCheckerService healthChecker) {
         MyNodeStatusUpdater nodeStatusUpdater = new MyNodeStatusUpdater(
-            context, dispatcher, healthChecker, metrics,
-            containerTokenSecretManager);
+            context, dispatcher, healthChecker, metrics);
         MyResourceTracker2 myResourceTracker2 = new MyResourceTracker2();
         myResourceTracker2.registerNodeAction = NodeAction.SHUTDOWN;
         nodeStatusUpdater.resourceTracker = myResourceTracker2;
@@ -556,22 +546,19 @@ public class TestNodeStatusUpdater {
     nm = new NodeManager() {
       @Override
       protected NodeStatusUpdater createNodeStatusUpdater(Context context,
-          Dispatcher dispatcher, NodeHealthCheckerService healthChecker,
-          ContainerTokenSecretManager containerTokenSecretManager) {
+          Dispatcher dispatcher, NodeHealthCheckerService healthChecker) {
         return new MyNodeStatusUpdater(context, dispatcher, healthChecker,
-                                       metrics, containerTokenSecretManager);
+                                       metrics);
       }
 
       @Override
       protected ContainerManagerImpl createContainerManager(Context context,
           ContainerExecutor exec, DeletionService del,
           NodeStatusUpdater nodeStatusUpdater,
-          ContainerTokenSecretManager containerTokenSecretManager,
           ApplicationACLsManager aclsManager,
           LocalDirsHandlerService diskhandler) {
-        return new ContainerManagerImpl(context, exec, del,
-            nodeStatusUpdater, metrics, containerTokenSecretManager,
-            aclsManager, diskhandler) {
+        return new ContainerManagerImpl(context, exec, del, nodeStatusUpdater,
+          metrics, aclsManager, diskhandler) {
           @Override
           public void start() {
             // Simulating failure of starting RPC server
@@ -654,11 +641,9 @@ public class TestNodeStatusUpdater {
     return new NodeManager() {
       @Override
       protected NodeStatusUpdater createNodeStatusUpdater(Context context,
-          Dispatcher dispatcher, NodeHealthCheckerService healthChecker,
-          ContainerTokenSecretManager containerTokenSecretManager) {
+          Dispatcher dispatcher, NodeHealthCheckerService healthChecker) {
         MyNodeStatusUpdater myNodeStatusUpdater = new MyNodeStatusUpdater(
-            context, dispatcher, healthChecker, metrics,
-            containerTokenSecretManager);
+            context, dispatcher, healthChecker, metrics);
         MyResourceTracker2 myResourceTracker2 = new MyResourceTracker2();
         myResourceTracker2.heartBeatNodeAction = nodeHeartBeatAction;
         myNodeStatusUpdater.resourceTracker = myResourceTracker2;

Modified: hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java Tue Oct 16 00:02:55 2012
@@ -54,8 +54,8 @@ import org.apache.hadoop.yarn.server.nod
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationState;
 import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
+import org.apache.hadoop.yarn.server.nodemanager.security.NMContainerTokenSecretManager;
 import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
-import org.apache.hadoop.yarn.server.security.ContainerTokenSecretManager;
 import org.apache.hadoop.yarn.service.Service.STATE;
 import org.junit.After;
 import org.junit.Before;
@@ -70,8 +70,6 @@ public abstract class BaseContainerManag
   protected static File localLogDir;
   protected static File remoteLogDir;
   protected static File tmpDir;
-  protected ContainerTokenSecretManager containerTokenSecretManager =
-      new ContainerTokenSecretManager(new Configuration());
 
   protected final NodeManagerMetrics metrics = NodeManagerMetrics.create();
 
@@ -93,7 +91,8 @@ public abstract class BaseContainerManag
       .getLog(BaseContainerManagerTest.class);
 
   protected Configuration conf = new YarnConfiguration();
-  protected Context context = new NMContext();
+  protected Context context = new NMContext(new NMContainerTokenSecretManager(
+    conf));
   protected ContainerExecutor exec;
   protected DeletionService delSrvc;
   protected String user = "nobody";
@@ -101,7 +100,7 @@ public abstract class BaseContainerManag
   protected LocalDirsHandlerService dirsHandler;
 
   protected NodeStatusUpdater nodeStatusUpdater = new NodeStatusUpdaterImpl(
-      context, new AsyncDispatcher(), null, metrics, this.containerTokenSecretManager) {
+      context, new AsyncDispatcher(), null, metrics) {
     @Override
     protected ResourceTracker getRMClient() {
       return new LocalRMInterface();
@@ -155,9 +154,9 @@ public abstract class BaseContainerManag
     nodeHealthChecker = new NodeHealthCheckerService();
     nodeHealthChecker.init(conf);
     dirsHandler = nodeHealthChecker.getDiskHandler();
-    containerManager = new ContainerManagerImpl(context, exec, delSrvc,
-        nodeStatusUpdater, metrics, this.containerTokenSecretManager,
-        new ApplicationACLsManager(conf), dirsHandler);
+    containerManager =
+        new ContainerManagerImpl(context, exec, delSrvc, nodeStatusUpdater,
+          metrics, new ApplicationACLsManager(conf), dirsHandler);
     containerManager.init(conf);
   }
 

Modified: hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java Tue Oct 16 00:02:55 2012
@@ -51,14 +51,13 @@ import org.apache.hadoop.yarn.api.record
 import org.apache.hadoop.yarn.api.records.URL;
 import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
 import org.apache.hadoop.yarn.server.nodemanager.CMgrCompletedAppsEvent;
-import org.apache.hadoop.yarn.server.nodemanager.DeletionService;
 import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.ExitCode;
 import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.Signal;
+import org.apache.hadoop.yarn.server.nodemanager.DeletionService;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationState;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceLocalizationService;
 import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
-import org.apache.hadoop.yarn.server.security.ContainerTokenSecretManager;
 import org.apache.hadoop.yarn.util.ConverterUtils;
 import org.junit.Test;
 
@@ -384,11 +383,9 @@ public class TestContainerManager extend
     delSrvc = new DeletionService(exec);
     delSrvc.init(conf);
 
-    ContainerTokenSecretManager containerTokenSecretManager = new 
-        ContainerTokenSecretManager(conf);
-    containerManager = new ContainerManagerImpl(context, exec, delSrvc,
-        nodeStatusUpdater, metrics, containerTokenSecretManager,
-        new ApplicationACLsManager(conf), dirsHandler);
+    containerManager =
+        new ContainerManagerImpl(context, exec, delSrvc, nodeStatusUpdater,
+          metrics, new ApplicationACLsManager(conf), dirsHandler);
     containerManager.init(conf);
     containerManager.start();
 

Modified: hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java Tue Oct 16 00:02:55 2012
@@ -30,9 +30,9 @@ import java.util.HashMap;
 import java.util.List;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
 import org.apache.hadoop.yarn.event.DrainDispatcher;
@@ -50,8 +50,8 @@ import org.apache.hadoop.yarn.server.nod
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ApplicationLocalizationEvent;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.LocalizationEvent;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.LocalizationEventType;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerEventType;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerEvent;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerEventType;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitorEvent;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitorEventType;
 import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;

Modified: hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResource.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResource.java?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResource.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResource.java Tue Oct 16 00:02:55 2012
@@ -37,7 +37,7 @@ import static org.junit.Assert.*;
 public class TestLocalResource {
 
   static org.apache.hadoop.yarn.api.records.LocalResource getYarnResource(Path p, long size,
-      long timestamp, LocalResourceType type, LocalResourceVisibility state)
+      long timestamp, LocalResourceType type, LocalResourceVisibility state, String pattern)
       throws URISyntaxException {
     org.apache.hadoop.yarn.api.records.LocalResource ret = RecordFactoryProvider.getRecordFactory(null).newRecordInstance(org.apache.hadoop.yarn.api.records.LocalResource.class);
     ret.setResource(ConverterUtils.getYarnUrlFromURI(p.toUri()));
@@ -45,6 +45,7 @@ public class TestLocalResource {
     ret.setTimestamp(timestamp);
     ret.setType(type);
     ret.setVisibility(state);
+    ret.setPattern(pattern);
     return ret;
   }
 
@@ -72,9 +73,9 @@ public class TestLocalResource {
 
     long basetime = r.nextLong() >>> 2;
     org.apache.hadoop.yarn.api.records.LocalResource yA = getYarnResource(
-        new Path("http://yak.org:80/foobar"), -1, basetime, FILE, PUBLIC);
+        new Path("http://yak.org:80/foobar"), -1, basetime, FILE, PUBLIC, null);
     org.apache.hadoop.yarn.api.records.LocalResource yB = getYarnResource(
-        new Path("http://yak.org:80/foobar"), -1, basetime, FILE, PUBLIC);
+        new Path("http://yak.org:80/foobar"), -1, basetime, FILE, PUBLIC, null);
     final LocalResourceRequest a = new LocalResourceRequest(yA);
     LocalResourceRequest b = new LocalResourceRequest(yA);
     checkEqual(a, b);
@@ -83,31 +84,37 @@ public class TestLocalResource {
 
     // ignore visibility
     yB = getYarnResource(
-        new Path("http://yak.org:80/foobar"), -1, basetime, FILE, PRIVATE);
+        new Path("http://yak.org:80/foobar"), -1, basetime, FILE, PRIVATE, null);
     b = new LocalResourceRequest(yB);
     checkEqual(a, b);
 
     // ignore size
     yB = getYarnResource(
-        new Path("http://yak.org:80/foobar"), 0, basetime, FILE, PRIVATE);
+        new Path("http://yak.org:80/foobar"), 0, basetime, FILE, PRIVATE, null);
     b = new LocalResourceRequest(yB);
     checkEqual(a, b);
 
     // note path
     yB = getYarnResource(
-        new Path("hdfs://dingo.org:80/foobar"), 0, basetime, ARCHIVE, PUBLIC);
+        new Path("hdfs://dingo.org:80/foobar"), 0, basetime, ARCHIVE, PUBLIC, null);
     b = new LocalResourceRequest(yB);
     checkNotEqual(a, b);
 
     // note type
     yB = getYarnResource(
-        new Path("http://yak.org:80/foobar"), 0, basetime, ARCHIVE, PUBLIC);
+        new Path("http://yak.org:80/foobar"), 0, basetime, ARCHIVE, PUBLIC, null);
     b = new LocalResourceRequest(yB);
     checkNotEqual(a, b);
 
     // note timestamp
     yB = getYarnResource(
-        new Path("http://yak.org:80/foobar"), 0, basetime + 1, FILE, PUBLIC);
+        new Path("http://yak.org:80/foobar"), 0, basetime + 1, FILE, PUBLIC, null);
+    b = new LocalResourceRequest(yB);
+    checkNotEqual(a, b);
+
+    // note pattern
+    yB = getYarnResource(
+        new Path("http://yak.org:80/foobar"), 0, basetime + 1, FILE, PUBLIC, "^/foo/.*");
     b = new LocalResourceRequest(yB);
     checkNotEqual(a, b);
   }
@@ -120,24 +127,35 @@ public class TestLocalResource {
     System.out.println("SEED: " + seed);
     long basetime = r.nextLong() >>> 2;
     org.apache.hadoop.yarn.api.records.LocalResource yA = getYarnResource(
-        new Path("http://yak.org:80/foobar"), -1, basetime, FILE, PUBLIC);
+        new Path("http://yak.org:80/foobar"), -1, basetime, FILE, PUBLIC, "^/foo/.*");
     final LocalResourceRequest a = new LocalResourceRequest(yA);
 
     // Path primary
     org.apache.hadoop.yarn.api.records.LocalResource yB = getYarnResource(
-        new Path("http://yak.org:80/foobaz"), -1, basetime, FILE, PUBLIC);
+        new Path("http://yak.org:80/foobaz"), -1, basetime, FILE, PUBLIC, "^/foo/.*");
     LocalResourceRequest b = new LocalResourceRequest(yB);
     assertTrue(0 > a.compareTo(b));
 
     // timestamp secondary
     yB = getYarnResource(
-        new Path("http://yak.org:80/foobar"), -1, basetime + 1, FILE, PUBLIC);
+        new Path("http://yak.org:80/foobar"), -1, basetime + 1, FILE, PUBLIC, "^/foo/.*");
     b = new LocalResourceRequest(yB);
     assertTrue(0 > a.compareTo(b));
 
     // type tertiary
     yB = getYarnResource(
-        new Path("http://yak.org:80/foobar"), -1, basetime, ARCHIVE, PUBLIC);
+        new Path("http://yak.org:80/foobar"), -1, basetime, ARCHIVE, PUBLIC, "^/foo/.*");
+    b = new LocalResourceRequest(yB);
+    assertTrue(0 != a.compareTo(b)); // don't care about order, just ne
+    
+    // path 4th
+    yB = getYarnResource(
+        new Path("http://yak.org:80/foobar"), -1, basetime, ARCHIVE, PUBLIC, "^/food/.*");
+    b = new LocalResourceRequest(yB);
+    assertTrue(0 != a.compareTo(b)); // don't care about order, just ne
+    
+    yB = getYarnResource(
+        new Path("http://yak.org:80/foobar"), -1, basetime, ARCHIVE, PUBLIC, null);
     b = new LocalResourceRequest(yB);
     assertTrue(0 != a.compareTo(b)); // don't care about order, just ne
   }

Modified: hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResourcesTrackerImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResourcesTrackerImpl.java?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResourcesTrackerImpl.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResourcesTrackerImpl.java Tue Oct 16 00:02:55 2012
@@ -230,7 +230,7 @@ public class TestLocalResourcesTrackerIm
       long ts, LocalResourceVisibility vis) {
     final LocalResourceRequest req =
         new LocalResourceRequest(new Path("file:///tmp/" + user + "/rsrc" + i),
-            ts + i * 2000, LocalResourceType.FILE, vis);
+            ts + i * 2000, LocalResourceType.FILE, vis, null);
     return req;
   }
 



Mime
View raw message