hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From vino...@apache.org
Subject svn commit: r1128990 - in /hadoop/mapreduce/branches/MR-279: ./ yarn/yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/ yar...
Date Mon, 30 May 2011 04:26:47 GMT
Author: vinodkv
Date: Mon May 30 04:26:46 2011
New Revision: 1128990

URL: http://svn.apache.org/viewvc?rev=1128990&view=rev
Log:
Fixing and reneabling TestContainerTokenSecretManager.

Modified:
    hadoop/mapreduce/branches/MR-279/CHANGES.txt
    hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerToken.java
    hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
    hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerTokenSecretManager.java

Modified: hadoop/mapreduce/branches/MR-279/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/CHANGES.txt?rev=1128990&r1=1128989&r2=1128990&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/CHANGES.txt (original)
+++ hadoop/mapreduce/branches/MR-279/CHANGES.txt Mon May 30 04:26:46 2011
@@ -4,6 +4,8 @@ Trunk (unreleased changes)
 
   MAPREDUCE-279
 
+    Fixing and reneabling TestContainerTokenSecretManager. (vinodkv)
+
     Set correct version of avro-maven-plugin that is available in
     apache maven repositories. (vinodkv)
   

Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerToken.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerToken.java?rev=1128990&r1=1128989&r2=1128990&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerToken.java
(original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerToken.java
Mon May 30 04:26:46 2011
@@ -1,3 +1,21 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package org.apache.hadoop.yarn.api.records;
 
 import java.nio.ByteBuffer;

Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java?rev=1128990&r1=1128989&r2=1128990&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
(original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
Mon May 30 04:26:46 2011
@@ -448,7 +448,8 @@ public class FifoScheduler implements Re
                 node.getHttpAddress(), capability);
         // If security is enabled, send the container-tokens too.
         if (UserGroupInformation.isSecurityEnabled()) {
-          ContainerToken containerToken = recordFactory.newRecordInstance(ContainerToken.class);
+          ContainerToken containerToken =
+              recordFactory.newRecordInstance(ContainerToken.class);
           ContainerTokenIdentifier tokenidentifier =
             new ContainerTokenIdentifier(container.getId(),
                 container.getContainerManagerAddress(), container.getResource());

Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerTokenSecretManager.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerTokenSecretManager.java?rev=1128990&r1=1128989&r2=1128990&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerTokenSecretManager.java
(original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerTokenSecretManager.java
Mon May 30 04:26:46 2011
@@ -20,6 +20,9 @@ package org.apache.hadoop.yarn.server;
 
 import static org.junit.Assert.fail;
 
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileWriter;
 import java.io.IOException;
 import java.net.InetSocketAddress;
 import java.security.PrivilegedAction;
@@ -35,9 +38,11 @@ import org.apache.hadoop.conf.Configurat
 import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
 import org.apache.hadoop.fs.FileContext;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.UnsupportedFileSystemException;
 import org.apache.hadoop.io.DataInputBuffer;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.AccessControlException;
 import org.apache.hadoop.security.SecurityInfo;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.token.Token;
@@ -56,11 +61,13 @@ import org.apache.hadoop.yarn.api.record
 import org.apache.hadoop.yarn.api.records.Container;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.ContainerToken;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.LocalResourceType;
+import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
 import org.apache.hadoop.yarn.api.records.Priority;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.api.records.ResourceRequest;
 import org.apache.hadoop.yarn.api.records.URL;
-import org.apache.hadoop.yarn.conf.YARNApplicationConstants;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
 import org.apache.hadoop.yarn.factories.RecordFactory;
@@ -73,23 +80,35 @@ import org.apache.hadoop.yarn.security.C
 import org.apache.hadoop.yarn.security.SchedulerSecurityInfo;
 import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
 import org.apache.hadoop.yarn.util.ConverterUtils;
-import org.junit.Ignore;
+import org.junit.BeforeClass;
 import org.junit.Test;
 
-@Ignore
 public class TestContainerTokenSecretManager {
 
   private static Log LOG = LogFactory
       .getLog(TestContainerTokenSecretManager.class);
-  private static final RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
+  private static final RecordFactory recordFactory = RecordFactoryProvider
+      .getRecordFactory(null);
+  private static FileContext localFS = null;
+  private static final File localDir = new File("target",
+      TestContainerTokenSecretManager.class.getName() + "-localDir")
+      .getAbsoluteFile();
+
+  @BeforeClass
+  public static void setup() throws AccessControlException,
+      FileNotFoundException, UnsupportedFileSystemException, IOException {
+    localFS = FileContext.getLocalFSFileContext();
+    localFS.delete(new Path(localDir.getAbsolutePath()), true);
+    localDir.mkdir();
+  }
 
   @Test
   public void test() throws IOException, InterruptedException {
-    final ContainerId containerID = recordFactory.newRecordInstance(ContainerId.class);
-    ApplicationId appID = recordFactory.newRecordInstance(ApplicationId.class);
+
+    final ApplicationId appID = recordFactory.newRecordInstance(ApplicationId.class);
     appID.setClusterTimestamp(1234);
     appID.setId(5);
-    containerID.setAppId(appID);
+
     final Configuration conf = new Configuration();
     conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
         "kerberos");
@@ -106,9 +125,11 @@ public class TestContainerTokenSecretMan
     final YarnRPC yarnRPC = YarnRPC.create(conf);
 
     // Submit an application
-    ApplicationSubmissionContext appSubmissionContext = recordFactory.newRecordInstance(ApplicationSubmissionContext.class);
-    appSubmissionContext.setApplicationId(containerID.getAppId());
-    appSubmissionContext.setMasterCapability(recordFactory.newRecordInstance(Resource.class));
+    ApplicationSubmissionContext appSubmissionContext =
+        recordFactory.newRecordInstance(ApplicationSubmissionContext.class);
+    appSubmissionContext.setApplicationId(appID);
+    appSubmissionContext.setMasterCapability(recordFactory
+        .newRecordInstance(Resource.class));
     appSubmissionContext.getMasterCapability().setMemory(1024);
 //    appSubmissionContext.resources = new HashMap<String, URL>();
     appSubmissionContext.setUser("testUser");
@@ -116,26 +137,35 @@ public class TestContainerTokenSecretMan
 //    appSubmissionContext.command = new ArrayList<String>();
     appSubmissionContext.addCommand("sleep");
     appSubmissionContext.addCommand("100");
-    URL yarnUrlForJobSubmitDir =
+
+    // TODO: Use a resource to work around bugs. Today NM doesn't create local
+    // app-dirs if there are no file to download!!
+    File file = new File(localDir.getAbsolutePath(), "testFile");
+    FileWriter tmpFile = new FileWriter(file);
+    tmpFile.write("testing");
+    tmpFile.close();
+    URL testFileURL =
         ConverterUtils.getYarnUrlFromPath(FileContext.getFileContext()
-            .makeQualified(new Path("testPath")));
-    appSubmissionContext.setResource(
-        YARNApplicationConstants.JOB_SUBMIT_DIR, yarnUrlForJobSubmitDir);
+            .makeQualified(new Path(localDir.getAbsolutePath(), "testFile")));
+    LocalResource rsrc = recordFactory.newRecordInstance(LocalResource.class);
+    rsrc.setResource(testFileURL);
+    rsrc.setSize(file.length());
+    rsrc.setTimestamp(file.lastModified());
+    rsrc.setType(LocalResourceType.FILE);
+    rsrc.setVisibility(LocalResourceVisibility.PRIVATE);
+    appSubmissionContext.setResourceTodo("testFile", rsrc);
     resourceManager.getApplicationsManager().submitApplication(
         appSubmissionContext);
 
     // Wait till container gets allocated for AM
     int waitCounter = 0;
     Application app =
-        resourceManager.getApplicationsManager().getApplication(
-            containerID.getAppId());
+        resourceManager.getApplicationsManager().getApplication(appID);
     while (app.getState() != ApplicationState.LAUNCHED && waitCounter <= 20) {
       Thread.sleep(1000);
       LOG.info("Waiting for AM to be allocated a container. Current state is "
           + app.getState());
-      app =
-          resourceManager.getApplicationsManager().getApplication(
-              containerID.getAppId());
+      app = resourceManager.getApplicationsManager().getApplication(appID);
     }
 
     Assert.assertTrue(ApplicationState.PENDING != app.getState());
@@ -149,7 +179,7 @@ public class TestContainerTokenSecretMan
     final InetSocketAddress schedulerAddr =
         NetUtils.createSocketAddr(schedulerAddressString);
     ApplicationTokenIdentifier appTokenIdentifier =
-        new ApplicationTokenIdentifier(containerID.getAppId());
+        new ApplicationTokenIdentifier(appID);
     ApplicationTokenSecretManager appTokenSecretManager =
         new ApplicationTokenSecretManager();
     appTokenSecretManager.setMasterKey(ApplicationTokenSecretManager
@@ -179,11 +209,11 @@ public class TestContainerTokenSecretMan
             .newRecordInstance(RegisterApplicationMasterRequest.class);
     ApplicationMaster applicationMaster = recordFactory
         .newRecordInstance(ApplicationMaster.class);
-    applicationMaster.setApplicationId(containerID.getAppId());
+    applicationMaster.setApplicationId(appID);
     applicationMaster.setState(ApplicationState.RUNNING);
     ApplicationStatus status =
         recordFactory.newRecordInstance(ApplicationStatus.class);
-    status.setApplicationId(containerID.getAppId());
+    status.setApplicationId(appID);
     applicationMaster.setStatus(status);
     request.setApplicationMaster(applicationMaster);
     scheduler.registerApplicationMaster(request);
@@ -200,7 +230,8 @@ public class TestContainerTokenSecretMan
     ask.add(rr);
     ArrayList<Container> release = new ArrayList<Container>();
     
-    AllocateRequest allocateRequest = recordFactory.newRecordInstance(AllocateRequest.class);
+    AllocateRequest allocateRequest =
+        recordFactory.newRecordInstance(AllocateRequest.class);
     allocateRequest.setApplicationStatus(status);
     allocateRequest.addAllAsks(ask);
     allocateRequest.addAllReleases(release);
@@ -215,7 +246,8 @@ public class TestContainerTokenSecretMan
       status.setResponseId(status.getResponseId() + 1);
       allocateRequest.setApplicationStatus(status);
       allocatedContainers =
-          scheduler.allocate(allocateRequest).getAMResponse().getContainerList();
+          scheduler.allocate(allocateRequest).getAMResponse()
+              .getContainerList();
     }
 
     Assert.assertNotNull("Container is not allocted!", allocatedContainers);
@@ -240,10 +272,17 @@ public class TestContainerTokenSecretMan
       public Void run() {
         ContainerManager client =
             (ContainerManager) yarnRPC.getProxy(ContainerManager.class,
-                NetUtils.createSocketAddr(allocatedContainer.getContainerManagerAddress()
-                    ), conf);
+                NetUtils.createSocketAddr(allocatedContainer
+                    .getContainerManagerAddress()), conf);
         try {
-          GetContainerStatusRequest request = recordFactory.newRecordInstance(GetContainerStatusRequest.class);
+          LOG.info("Going to make a getContainerStatus() legal request");
+          GetContainerStatusRequest request =
+              recordFactory
+                  .newRecordInstance(GetContainerStatusRequest.class);
+          ContainerId containerID =
+              recordFactory.newRecordInstance(ContainerId.class);
+          containerID.setAppId(appID);
+          containerID.setId(1);
           request.setContainerId(containerID);
           client.getContainerStatus(request);
         } catch (YarnRemoteException e) {
@@ -277,12 +316,37 @@ public class TestContainerTokenSecretMan
     maliceUser.doAs(new PrivilegedAction<Void>() {
       @Override
       public Void run() {
+        ContainerManager client =
+            (ContainerManager) yarnRPC.getProxy(ContainerManager.class,
+                NetUtils.createSocketAddr(allocatedContainer
+                    .getContainerManagerAddress()), conf);
+        ContainerId containerID;
+
+        LOG.info("Going to contact NM:  ilLegal request");
+        GetContainerStatusRequest request =
+              recordFactory
+                  .newRecordInstance(GetContainerStatusRequest.class);
+        containerID =
+              recordFactory.newRecordInstance(ContainerId.class);
+        containerID.setAppId(appID);
+        containerID.setId(1);
+        request.setContainerId(containerID);
         try {
-          yarnRPC.getProxy(ContainerManager.class, NetUtils
-              .createSocketAddr(allocatedContainer.getContainerManagerAddress()), conf);
-          fail("Connection initiation with illegally modified tokens is expected to fail.");
-        } catch (YarnException e) {
-          LOG.info("Error", e);
+          client.getContainerStatus(request);
+          fail("Connection initiation with illegally modified "
+              + "tokens is expected to fail.");
+        } catch (YarnRemoteException e) {
+          LOG.error("Got exception", e);
+          fail("Cannot get a YARN remote exception as " +
+          		"it will indicate RPC success");
+        } catch (Exception e) {
+          Assert.assertEquals(
+              java.lang.reflect.UndeclaredThrowableException.class
+                  .getCanonicalName(), e.getClass().getCanonicalName());
+          Assert
+              .assertEquals(
+                  "DIGEST-MD5: digest response format violation. Mismatched response.",
+                  e.getCause().getCause().getMessage());
         }
         return null;
       }



Mime
View raw message