hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From vino...@apache.org
Subject svn commit: r1099337 [3/3] - in /hadoop/mapreduce/branches/MR-279: ./ mr-client/hadoop-mapreduce-client-app/ mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/ mr-client/hadoop-mapreduce-client-shuffle/src/...
Date Wed, 04 May 2011 06:53:53 GMT
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/NodeManagerImpl.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/NodeManagerImpl.java?rev=1099337&r1=1099336&r2=1099337&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/NodeManagerImpl.java
(original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/NodeManagerImpl.java
Wed May  4 06:53:52 2011
@@ -80,8 +80,7 @@ public class NodeManagerImpl implements 
   
   /* set of containers that need to be cleaned */
   private final Set<Container> containersToClean = 
-    new TreeSet<Container>(new org.apache.hadoop.yarn.server.resourcemanager.resource.Container.ContainerComparator());
-
+    new TreeSet<Container>(new org.apache.hadoop.yarn.util.BuilderUtils.ContainerComparator());
   
   /* the list of applications that have finished and need to be purged */
   private final List<ApplicationId> finishedApplications = new ArrayList<ApplicationId>();

Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java?rev=1099337&r1=1099336&r2=1099337&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
(original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
Wed May  4 06:53:52 2011
@@ -56,6 +56,7 @@ import org.apache.hadoop.yarn.server.res
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Application;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
 import org.apache.hadoop.yarn.server.security.ContainerTokenSecretManager;
+import org.apache.hadoop.yarn.util.BuilderUtils;
 
 @LimitedPrivate("yarn")
 @Evolving
@@ -99,12 +100,12 @@ implements ResourceScheduler, CapacitySc
   private Resource clusterResource = 
     RecordFactoryProvider.getRecordFactory(null).newRecordInstance(Resource.class);
   private int numNodeManagers = 0;
-  
+
   private Resource minimumAllocation;
 
   private Map<ApplicationId, Application> applications = 
     new TreeMap<ApplicationId, Application>(
-        new org.apache.hadoop.yarn.util.BuilderUtils.ApplicationIdComparator());
+        new BuilderUtils.ApplicationIdComparator());
 
   private boolean initialized = false;
 

Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java?rev=1099337&r1=1099336&r2=1099337&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
(original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
Wed May  4 06:53:52 2011
@@ -56,6 +56,7 @@ import org.apache.hadoop.yarn.server.res
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
 import org.apache.hadoop.yarn.server.security.ContainerTokenSecretManager;
+import org.apache.hadoop.yarn.util.BuilderUtils;
 
 @Private
 @Unstable
@@ -826,25 +827,24 @@ public class LeafQueue implements Queue 
                                                                 // crash the
                                                                 // scheduler.
     
-      if (availableContainers > 0) {
-
-        List<Container> containers =
-          new ArrayList<Container>();
-        Container container =
-          org.apache.hadoop.yarn.server.resourcemanager.resource.Container
-          .create(recordFactory, application.getApplicationId(),
+    if (availableContainers > 0) {
+      List<Container> containers =
+        new ArrayList<Container>();
+      Container container =
+          BuilderUtils.newContainer(this.recordFactory,
+              application.getApplicationId(),
               application.getNewContainerId(), node.getNodeAddress(),
               node.getHttpAddress(), capability);
-
-        // If security is enabled, send the container-tokens too.
-        if (UserGroupInformation.isSecurityEnabled()) {
-          ContainerToken containerToken = RecordFactoryProvider.getRecordFactory(null).newRecordInstance(ContainerToken.class);
-          ContainerTokenIdentifier tokenidentifier =
-            new ContainerTokenIdentifier(container.getId(),
-                container.getContainerManagerAddress(), container.getResource());
-          containerToken.setIdentifier(ByteBuffer.wrap(tokenidentifier.getBytes()));
-          containerToken.setKind(ContainerTokenIdentifier.KIND.toString());
-          containerToken.setPassword(ByteBuffer.wrap(containerTokenSecretManager
+      
+      // If security is enabled, send the container-tokens too.
+      if (UserGroupInformation.isSecurityEnabled()) {
+        ContainerToken containerToken = this.recordFactory.newRecordInstance(ContainerToken.class);
+        ContainerTokenIdentifier tokenidentifier =
+          new ContainerTokenIdentifier(container.getId(),
+              container.getContainerManagerAddress(), container.getResource());
+        containerToken.setIdentifier(ByteBuffer.wrap(tokenidentifier.getBytes()));
+        containerToken.setKind(ContainerTokenIdentifier.KIND.toString());
+        containerToken.setPassword(ByteBuffer.wrap(containerTokenSecretManager
               .createPassword(tokenidentifier)));
           containerToken.setService(container.getContainerManagerAddress());
           container.setContainerToken(containerToken);
@@ -1019,4 +1019,4 @@ public class LeafQueue implements Queue 
     parent.recoverContainer(clusterResource, application, container);
 
   }
-}
\ No newline at end of file
+}

Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java?rev=1099337&r1=1099336&r2=1099337&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
(original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
Wed May  4 06:53:52 2011
@@ -51,10 +51,10 @@ import org.apache.hadoop.yarn.factory.pr
 import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
 import org.apache.hadoop.yarn.server.resourcemanager.applicationsmanager.events.ASMEvent;
 import org.apache.hadoop.yarn.server.resourcemanager.applicationsmanager.events.ApplicationMasterEvents.ApplicationTrackerEventType;
-import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources;
 import org.apache.hadoop.yarn.server.resourcemanager.recovery.ApplicationsStore.ApplicationStore;
 import org.apache.hadoop.yarn.server.resourcemanager.recovery.Store.ApplicationInfo;
 import org.apache.hadoop.yarn.server.resourcemanager.recovery.Store.RMState;
+import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources;
 import org.apache.hadoop.yarn.server.resourcemanager.resourcetracker.ClusterTracker;
 import org.apache.hadoop.yarn.server.resourcemanager.resourcetracker.NodeInfo;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Application;
@@ -64,6 +64,7 @@ import org.apache.hadoop.yarn.server.res
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
 import org.apache.hadoop.yarn.server.security.ContainerTokenSecretManager;
+import org.apache.hadoop.yarn.util.BuilderUtils;
 
 @LimitedPrivate("yarn")
 @Evolving
@@ -86,9 +87,9 @@ public class FifoScheduler implements Re
   public static final Resource MINIMUM_ALLOCATION =
     Resources.createResource(MINIMUM_MEMORY);
 
-  Map<ApplicationId, Application> applications = 
-    new TreeMap<ApplicationId, Application>(
-        new org.apache.hadoop.yarn.util.BuilderUtils.ApplicationIdComparator());
+  Map<ApplicationId, Application> applications =
+      new TreeMap<ApplicationId, Application>(
+          new BuilderUtils.ApplicationIdComparator());
 
   private static final String DEFAULT_QUEUE_NAME = "default";
   private final QueueMetrics metrics =
@@ -437,10 +438,10 @@ public class FifoScheduler implements Re
         new ArrayList<Container>(assignedContainers);
       for (int i=0; i < assignedContainers; ++i) {
         Container container =
-          org.apache.hadoop.yarn.server.resourcemanager.resource.Container
-          .create(recordFactory, application.getApplicationId(),
-              application.getNewContainerId(), node.getNodeAddress(),
-              node.getHttpAddress(), capability);
+            BuilderUtils.newContainer(recordFactory,
+                application.getApplicationId(),
+                application.getNewContainerId(), node.getNodeAddress(),
+                node.getHttpAddress(), capability);
         // If security is enabled, send the container-tokens too.
         if (UserGroupInformation.isSecurityEnabled()) {
           ContainerToken containerToken = recordFactory.newRecordInstance(ContainerToken.class);

Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java?rev=1099337&r1=1099336&r2=1099337&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java
(original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java
Wed May  4 06:53:52 2011
@@ -81,7 +81,7 @@ public class Application {
         new org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceRequest.Comparator());
   final private Set<Container> release = 
     new TreeSet<Container>(
-        new org.apache.hadoop.yarn.server.resourcemanager.resource.Container.ContainerComparator());
+        new org.apache.hadoop.yarn.util.BuilderUtils.ContainerComparator());
 
   final private Map<String, NodeManager> nodes = 
     new HashMap<String, NodeManager>();
@@ -274,7 +274,7 @@ public class Application {
     for (Container container : response) {
       if (container.getState() != ContainerState.COMPLETE) {
         containers.add(
-            org.apache.hadoop.yarn.server.resourcemanager.resource.Container.create(
+            org.apache.hadoop.yarn.util.BuilderUtils.clone(
                 container));
       }
     }

Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java?rev=1099337&r1=1099336&r2=1099337&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java
(original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java
Wed May  4 06:53:52 2011
@@ -56,6 +56,7 @@ import org.apache.hadoop.yarn.server.api
 import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources;
 import org.apache.hadoop.yarn.server.resourcemanager.resourcetracker.NodeInfo;
 import org.apache.hadoop.yarn.server.resourcemanager.resourcetracker.RMResourceTrackerImpl;
+import org.apache.hadoop.yarn.util.BuilderUtils;
 
 @Private
 public class NodeManager implements ContainerManager {
@@ -165,11 +166,11 @@ public class NodeManager implements Cont
       }
     }
 
-    Container container = 
-        org.apache.hadoop.yarn.server.resourcemanager.resource.Container
-            .create(containerLaunchContext.getContainerId(),
-                containerManagerAddress, nodeHttpAddress,
-                containerLaunchContext.getResource());
+    Container container =
+        BuilderUtils.newContainer(containerLaunchContext.getContainerId(),
+            containerManagerAddress, nodeHttpAddress,
+            containerLaunchContext.getResource());
+
     applicationContainers.add(container);
     
     Resources.subtractFrom(available, containerLaunchContext.getResource());

Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestApplicationCleanup.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestApplicationCleanup.java?rev=1099337&r1=1099336&r2=1099337&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestApplicationCleanup.java
(original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestApplicationCleanup.java
Wed May  4 06:53:52 2011
@@ -25,8 +25,6 @@ import java.util.List;
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicInteger;
 
-
-import junit.framework.Assert;
 import junit.framework.TestCase;
 
 import org.apache.commons.logging.Log;
@@ -41,16 +39,12 @@ import org.apache.hadoop.yarn.api.record
 import org.apache.hadoop.yarn.api.records.Container;
 import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.api.records.Priority;
-import org.apache.hadoop.yarn.api.records.ResourceRequest;
 import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.api.records.YarnClusterMetrics;
+import org.apache.hadoop.yarn.api.records.ResourceRequest;
 import org.apache.hadoop.yarn.event.EventHandler;
 import org.apache.hadoop.yarn.factories.RecordFactory;
 import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
 import org.apache.hadoop.yarn.security.ApplicationTokenSecretManager;
-import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatRequest;
-import org.apache.hadoop.yarn.server.api.records.NodeHealthStatus;
-import org.apache.hadoop.yarn.server.api.records.NodeStatus;
 import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
 import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager.RMContext;
 import org.apache.hadoop.yarn.server.resourcemanager.applicationsmanager.events.ASMEvent;
@@ -58,29 +52,26 @@ import org.apache.hadoop.yarn.server.res
 import org.apache.hadoop.yarn.server.resourcemanager.applicationsmanager.events.ApplicationMasterEvents.ApplicationEventType;
 import org.apache.hadoop.yarn.server.resourcemanager.applicationsmanager.events.ApplicationMasterEvents.ApplicationTrackerEventType;
 import org.apache.hadoop.yarn.server.resourcemanager.applicationsmanager.events.ApplicationMasterEvents.SNEventType;
-import org.apache.hadoop.yarn.server.resourcemanager.recovery.ApplicationsStore.ApplicationStore;
 import org.apache.hadoop.yarn.server.resourcemanager.recovery.MemStore;
-import org.apache.hadoop.yarn.server.resourcemanager.recovery.Store.RMState;
-import org.apache.hadoop.yarn.server.resourcemanager.resourcetracker.ClusterTracker;
 import org.apache.hadoop.yarn.server.resourcemanager.resourcetracker.NodeInfo;
 import org.apache.hadoop.yarn.server.resourcemanager.resourcetracker.NodeInfoTracker;
 import org.apache.hadoop.yarn.server.resourcemanager.resourcetracker.RMResourceTrackerImpl;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeManager;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeManagerImpl;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeResponse;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceListener;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler;
 import org.apache.hadoop.yarn.server.security.ContainerTokenSecretManager;
 import org.junit.After;
 import org.junit.Before;
+import org.junit.Ignore;
 import org.junit.Test;
 
 /**
  * Testing application cleanup (notifications to nodemanagers).
  *
  */
+@Ignore
 public class TestApplicationCleanup extends TestCase {
   private static final Log LOG = LogFactory.getLog(TestApplicationCleanup.class);
   private static RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);

Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java?rev=1099337&r1=1099336&r2=1099337&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
(original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
Wed May  4 06:53:52 2011
@@ -149,9 +149,14 @@ public class MiniYARNCluster extends Com
         File logDir =
             new File(testWorkDir, MiniYARNCluster.this.getName()
                 + "-logDir");
+        File remoteLogDir =
+          new File(testWorkDir, MiniYARNCluster.this.getName()
+              + "-remoteLogDir");
         logDir.mkdir();
+        remoteLogDir.mkdir();
         LOG.info("Created logDir in " + logDir.getAbsolutePath());
         getConfig().set(NMConfig.NM_LOG_DIR, logDir.getAbsolutePath());
+        getConfig().set(NMConfig.REMOTE_USER_LOG_DIR, remoteLogDir.getAbsolutePath());
         nodeManager = new NodeManager() {
 
           @Override



Mime
View raw message