hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sur...@apache.org
Subject svn commit: r1399953 - in /hadoop/common/branches/branch-1-win: ./ src/core/org/apache/hadoop/fs/ src/mapred/org/apache/hadoop/mapred/ src/test/org/apache/hadoop/mapred/ src/test/testshell/
Date Fri, 19 Oct 2012 02:46:17 GMT
Author: suresh
Date: Fri Oct 19 02:46:16 2012
New Revision: 1399953

URL: http://svn.apache.org/viewvc?rev=1399953&view=rev
Log:
HADOOP-8899. Classpath exceeds maximum Windows command limit. Contributed by Ahmed El Baz.

Modified:
    hadoop/common/branches/branch-1-win/CHANGES.branch-1-win.txt
    hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/fs/FileUtil.java
    hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/DefaultTaskController.java
    hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/JvmManager.java
    hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/LinuxTaskController.java
    hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/TaskController.java
    hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/TaskRunner.java
    hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/mapred/TestJvmManager.java
    hadoop/common/branches/branch-1-win/src/test/testshell/ExternalMapReduce.java

Modified: hadoop/common/branches/branch-1-win/CHANGES.branch-1-win.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/CHANGES.branch-1-win.txt?rev=1399953&r1=1399952&r2=1399953&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/CHANGES.branch-1-win.txt (original)
+++ hadoop/common/branches/branch-1-win/CHANGES.branch-1-win.txt Fri Oct 19 02:46:16 2012
@@ -172,3 +172,6 @@ Branch-hadoop-1-win - unreleased
 
     HDFS-4065. TestDFSShell.testGet sporadically fails attempting to corrupt 
     block files due to race condition. (Chris Nauroth via suresh)
+
+    HADOOP-8899. Classpath exceeds maximum Windows command limit.
+    (Ahmed El Baz via suresh)

Modified: hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/fs/FileUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/fs/FileUtil.java?rev=1399953&r1=1399952&r2=1399953&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/fs/FileUtil.java (original)
+++ hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/fs/FileUtil.java Fri Oct
19 02:46:16 2012
@@ -19,7 +19,12 @@
 package org.apache.hadoop.fs;
 
 import java.io.*;
+import java.net.URL;
 import java.util.Enumeration;
+import java.util.List;
+import java.util.jar.Attributes;
+import java.util.jar.JarOutputStream;
+import java.util.jar.Manifest;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipFile;
 import org.apache.commons.logging.Log;
@@ -818,4 +823,37 @@ public class FileUtil {
     }
     return fileNames;
   }  
+  
+  /**
+   * Create a JAR file under at the given path, referencing all entries in the classpath
list.
+   * @param jarFile file to create with classpath entries in its manifest
+   * @param classPaths entries to be added in the manifest of the created Jar 
+   * @return jarFile created with classpath entries
+   */
+  public static File createJarWithClassPath(File jarFile, List<String> classPaths)

+      throws IOException {
+    StringBuffer jarClsPath = new StringBuffer();
+
+    // Append all entries in classpath list to the Jar
+    for (String clsEntry : classPaths) {
+      URL fileUrl = new URL(new File(clsEntry).toURI().toString());
+      jarClsPath.append(fileUrl.toExternalForm());
+      jarClsPath.append(" ");
+    }
+
+    // Create the manifest
+    Manifest jarManifest = new Manifest();
+    jarManifest.getMainAttributes().putValue(
+        Attributes.Name.MANIFEST_VERSION.toString(), "1.0");
+    
+    jarManifest.getMainAttributes().putValue(
+        Attributes.Name.CLASS_PATH.toString(), jarClsPath.toString().trim());
+
+    // Write the manifest to output JAR file
+    JarOutputStream jarStream = new JarOutputStream(
+        new FileOutputStream(jarFile), jarManifest);
+    
+    jarStream.close();
+    return jarFile;
+  }
 }

Modified: hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/DefaultTaskController.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/DefaultTaskController.java?rev=1399953&r1=1399952&r2=1399953&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/DefaultTaskController.java
(original)
+++ hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/DefaultTaskController.java
Fri Oct 19 02:46:16 2012
@@ -34,6 +34,7 @@ import org.apache.hadoop.mapreduce.serve
 import org.apache.hadoop.mapred.TaskTracker.LocalStorage;
 import org.apache.hadoop.util.ProcessTree.Signal;
 import org.apache.hadoop.util.ProcessTree;
+import org.apache.hadoop.util.StringUtils;
 import org.apache.hadoop.util.Shell.ShellCommandExecutor;
 
 import org.apache.commons.logging.Log;
@@ -55,6 +56,8 @@ public class DefaultTaskController exten
 
   private static final Log LOG = 
       LogFactory.getLog(DefaultTaskController.class);
+  // Prefix of the Jar file containing the classpath
+  protected static final String JAR_CLASSPATH_PREFIX = "classpath-";
   private FileSystem fs;
   @Override
   public void setConf(Configuration conf) {
@@ -88,6 +91,38 @@ public class DefaultTaskController exten
                                   File currentWorkDirectory,
                                   String stdout,
                                   String stderr) throws IOException {
+    // Assume the classpath is already given in setup or jvmArguments
+    return launchTask(user, jobId, attemptId, setup, jvmArguments, null,
+                      currentWorkDirectory, stdout, stderr);
+  }
+  
+  /**
+   * Create all of the directories for the task and launches the child jvm.
+   * Uses all items in the classpaths list as the classpath for the task to start
+   * If classPaths is not provided, then it is assumed it is already set as one
+   * of the setup steps, or in jvmArguments
+   * @param user the user name
+   * @param jobId the jobId in question
+   * @param attemptId the attempt id (cleanup attempts have .cleanup suffix)
+   * @param setup list of shell commands to execute before the jvm
+   * @param jvmArguments list of jvm arguments
+   * @param classpaths list of classpath items
+   * @param currentWorkDirectory the full path of the cwd for the task
+   * @param stdout the file to redirect stdout to
+   * @param stderr the file to redirect stderr to
+   * @return the exit code for the task
+   * @throws IOException
+   */
+  @Override
+  public int launchTask(String user, 
+                                  String jobId,
+                                  String attemptId,
+                                  List<String> setup,
+                                  List<String> jvmArguments,
+                                  List<String> classPaths,
+                                  File currentWorkDirectory,
+                                  String stdout,
+                                  String stderr) throws IOException {
     ShellCommandExecutor shExec = null;
     try {    	            
       FileSystem localFs = FileSystem.getLocal(getConf());
@@ -109,6 +144,30 @@ public class DefaultTaskController exten
         throw new IOException("Mkdirs failed to create " 
                    + logLocation);
       }
+      
+      //If provided, use all items in the classpaths list as the classpath for
+      //the task to start
+      if (classPaths != null && classPaths.size() > 0) {
+        String clsPaths;
+          
+        if (Shell.WINDOWS) {
+          // Prepare the path of the Jar file to create
+          File jarFile = File.createTempFile(JAR_CLASSPATH_PREFIX, ".jar", 
+                                             currentWorkDirectory.getParentFile());
+            
+          // Create a Jar file referencing all entries in the classPaths list.
+          clsPaths = FileUtil.createJarWithClassPath(jarFile, classPaths).
+                              getCanonicalPath();
+        } else {
+          // Get the classpath string
+          clsPaths = StringUtils.join(File.pathSeparator, classPaths);  
+        }
+
+        // Add the classpath as the first Java argument
+        jvmArguments.add(1, "-classpath");
+        jvmArguments.add(2, clsPaths);
+      }
+      
       //read the configuration for the job
       FileSystem rawFs = FileSystem.getLocal(getConf()).getRaw();
       long logSize = 0; //TODO MAPREDUCE-1100

Modified: hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/JvmManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/JvmManager.java?rev=1399953&r1=1399952&r2=1399953&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/JvmManager.java
(original)
+++ hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/JvmManager.java
Fri Oct 19 02:46:16 2012
@@ -52,9 +52,16 @@ class JvmManager {
   private JvmManagerForType reduceJvmManager;
   
   public JvmEnv constructJvmEnv(List<String> setup, Vector<String>vargs,
+      List<String> classPathsList, File stdout, File stderr, long logSize, 
+      File workDir, JobConf conf) {
+    return new JvmEnv(setup,vargs,classPathsList,stdout,stderr,logSize,
+        workDir,conf);
+  }
+  
+  public JvmEnv constructJvmEnv(List<String> setup, Vector<String>vargs,
       File stdout,File stderr,long logSize, File workDir, 
       JobConf conf) {
-    return new JvmEnv(setup,vargs,stdout,stderr,logSize,workDir,conf);
+    return constructJvmEnv(setup,vargs,null,stdout,stderr,logSize,workDir,conf);
   }
   
   public JvmManager(TaskTracker tracker) {
@@ -497,7 +504,7 @@ class JvmManager {
                   taskAttemptId.toString(); 
                 exitCode = tracker.getTaskController().launchTask(user,
                     jvmId.jobId.toString(), taskAttemptIdStr, env.setup,
-                    env.vargs, env.workDir, env.stdout.toString(),
+                    env.vargs, env.classpaths, env.workDir, env.stdout.toString(),
                     env.stderr.toString());
           }
         } catch (IOException ioe) {
@@ -592,6 +599,7 @@ class JvmManager {
   static class JvmEnv { //Helper class
     List<String> vargs;
     List<String> setup;
+    List<String> classpaths;
     File stdout;
     File stderr;
     File workDir;
@@ -601,12 +609,19 @@ class JvmManager {
 
     public JvmEnv(List <String> setup, Vector<String> vargs, File stdout, 
         File stderr, long logSize, File workDir, JobConf conf) {
+      this(setup, vargs, null, stdout, stderr, logSize, workDir, conf);
+    }
+        
+    // Construct JvmEnv with classpaths to use
+    public JvmEnv(List <String> setup, Vector<String> vargs, List <String>
classpaths, 
+        File stdout, File stderr, long logSize, File workDir, JobConf conf) {
       this.setup = setup;
       this.vargs = vargs;
       this.stdout = stdout;
       this.stderr = stderr;
       this.workDir = workDir;
       this.conf = conf;
+      this.classpaths = classpaths;
     }
   }
 }

Modified: hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/LinuxTaskController.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/LinuxTaskController.java?rev=1399953&r1=1399952&r2=1399953&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/LinuxTaskController.java
(original)
+++ hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/LinuxTaskController.java
Fri Oct 19 02:46:16 2012
@@ -204,9 +204,31 @@ class LinuxTaskController extends TaskCo
                                   File currentWorkDirectory,
                                   String stdout,
                                   String stderr) throws IOException {
+	return launchTask(user, jobId, attemptId, setup, jvmArguments,
+                      null, currentWorkDirectory, stdout, stderr);
+  }
 
+  @Override
+  public int launchTask(String user, 
+                                  String jobId,
+                                  String attemptId,
+                                  List<String> setup,
+                                  List<String> jvmArguments,
+                                  List<String> classPaths,
+                                  File currentWorkDirectory,
+                                  String stdout,
+                                  String stderr) throws IOException {
     ShellCommandExecutor shExec = null;
     try {
+      if (classPaths != null && classPaths.size() > 0) {
+        // Get the classpath string
+        String clsPaths = StringUtils.join(File.pathSeparator, classPaths);
+
+        // Add the classpath as the first Java argument
+        jvmArguments.add(1, "-classpath");
+        jvmArguments.add(2, clsPaths);
+      }
+
       FileSystem rawFs = FileSystem.getLocal(getConf()).getRaw();
       long logSize = 0; //TODO MAPREDUCE-1100
       // get the JVM command line.
@@ -259,7 +281,7 @@ class LinuxTaskController extends TaskCo
     }
     return 0;
   }
-
+  
   @Override
   public void createLogDir(TaskAttemptID taskID,
                            boolean isCleanup) throws IOException {

Modified: hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/TaskController.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/TaskController.java?rev=1399953&r1=1399952&r2=1399953&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/TaskController.java
(original)
+++ hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/TaskController.java
Fri Oct 19 02:46:16 2012
@@ -63,7 +63,7 @@ public abstract class TaskController imp
 
   final public static FsPermission TASK_LAUNCH_SCRIPT_PERMISSION =
   FsPermission.createImmutable((short) 0700); // rwx--------
-  
+
   public Configuration getConf() {
     return conf;
   }
@@ -127,6 +127,34 @@ public abstract class TaskController imp
                  String stderr) throws IOException;
   
   /**
+   * Create all of the directories for the task and launches the child jvm.
+   * Uses all items in the classpaths list as the classpath for the task to start
+   * If classPaths is not provided, then it is assumed it is already set as one
+   * of the setup steps, or in jvmArguments
+   * @param user the user name
+   * @param jobId the jobId in question
+   * @param attemptId the attempt id (cleanup attempts have .cleanup suffix)
+   * @param setup list of shell commands to execute before the jvm
+   * @param jvmArguments list of jvm arguments
+   * @param classpaths list of classpath items
+   * @param currentWorkDirectory the full path of the cwd for the task
+   * @param stdout the file to redirect stdout to
+   * @param stderr the file to redirect stderr to
+   * @return the exit code for the task
+   * @throws IOException
+   */
+  public abstract
+  int launchTask(String user, 
+                 String jobId,
+                 String attemptId,
+                 List<String> setup,
+                 List<String> jvmArguments,
+                 List<String> classPaths,
+                 File currentWorkDirectory,
+                 String stdout,
+                 String stderr) throws IOException;
+  
+  /**
    * Send a signal to a task pid as the user.
    * @param user the user name
    * @param taskPid the pid of the task

Modified: hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/TaskRunner.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/TaskRunner.java?rev=1399953&r1=1399952&r2=1399953&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/TaskRunner.java
(original)
+++ hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/TaskRunner.java
Fri Oct 19 02:46:16 2012
@@ -78,8 +78,6 @@ abstract class TaskRunner extends Thread
   
   static final String HADOOP_WORK_DIR = "HADOOP_WORK_DIR";
   
-  static final String JAVA_CLASSPATH = "CLASSPATH";
-  
   static final String MAPRED_ADMIN_USER_ENV =
     "mapreduce.admin.user.env";
 
@@ -219,8 +217,6 @@ abstract class TaskRunner extends Thread
       // Accumulates class paths for child.
       List<String> classPathsList = getClassPaths(conf, workDir,
                                                   taskDistributedCacheManager);
-      String classPaths = StringUtils.join(SYSTEM_PATH_SEPARATOR,
-                                           classPathsList);
 
       long logSize = TaskLog.getTaskLogLength(conf);
       
@@ -239,8 +235,8 @@ abstract class TaskRunner extends Thread
                  stderr);
       
       Map<String, String> env = new HashMap<String, String>();
-      errorInfo = getVMEnvironment(errorInfo, user, workDir, classPaths,
-                                   conf, env, taskid, logSize);
+      errorInfo = getVMEnvironment(errorInfo, user, workDir, conf, env,  
+                                   taskid, logSize);
       
       // flatten the env as a set of export commands
       List <String> setupCmds = new ArrayList<String>();
@@ -255,7 +251,7 @@ abstract class TaskRunner extends Thread
       }
       setupCmds.add(setup);
       
-      launchJvmAndWait(setupCmds, vargs, stdout, stderr, logSize, workDir);
+      launchJvmAndWait(setupCmds, vargs, classPathsList, stdout, stderr, logSize, workDir);
       tracker.getTaskTrackerInstrumentation().reportTaskEnd(t.getTaskID());
       if (exitCodeSet) {
         if (!killed && exitCode != 0) {
@@ -293,11 +289,12 @@ abstract class TaskRunner extends Thread
     }
   }
 
-  void launchJvmAndWait(List <String> setup, Vector<String> vargs, File stdout,
-      File stderr, long logSize, File workDir)
+  void launchJvmAndWait(List <String> setup, Vector<String> vargs, 
+	  List<String> classPathsList, File stdout, File stderr, long logSize, 
+	  File workDir)
       throws InterruptedException, IOException {
-    jvmManager.launchJvm(this, jvmManager.constructJvmEnv(setup, vargs, stdout,
-        stderr, logSize, workDir, conf));
+    jvmManager.launchJvm(this, jvmManager.constructJvmEnv(setup, vargs, classPathsList,
+        stdout, stderr, logSize, workDir, conf));
     synchronized (lock) {
       while (!done) {
         lock.wait();
@@ -305,6 +302,12 @@ abstract class TaskRunner extends Thread
     }
   }
 
+  void launchJvmAndWait(List <String> setup, Vector<String> vargs, File stdout,
+      File stderr, long logSize, File workDir)
+      throws InterruptedException, IOException {
+    launchJvmAndWait(setup, vargs, null, stdout, stderr, logSize, workDir);
+  }
+  
   /**
    * Prepare the log files for the task
    * 
@@ -568,9 +571,8 @@ abstract class TaskRunner extends Thread
   }
 
   private String getVMEnvironment(String errorInfo, String user, File workDir,
-                                  String classPaths, JobConf conf,
-                                  Map<String, String> env, TaskAttemptID taskid,
-                                  long logSize
+                                  JobConf conf, Map<String, String> env, 
+                                  TaskAttemptID taskid, long logSize
                                   ) throws Throwable {
     StringBuffer ldLibraryPath = new StringBuffer();
     ldLibraryPath.append(workDir.toString());
@@ -582,7 +584,6 @@ abstract class TaskRunner extends Thread
     }
     env.put("LD_LIBRARY_PATH", ldLibraryPath.toString());
     env.put(HADOOP_WORK_DIR, workDir.toString());
-    env.put(JAVA_CLASSPATH, classPaths.toString());
     //update user configured login-shell properties
     updateUserLoginEnv(errorInfo, user, conf, env);
     // put jobTokenFile name into env

Modified: hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/mapred/TestJvmManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/mapred/TestJvmManager.java?rev=1399953&r1=1399952&r2=1399953&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/mapred/TestJvmManager.java
(original)
+++ hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/mapred/TestJvmManager.java
Fri Oct 19 02:46:16 2012
@@ -22,8 +22,12 @@ import java.io.File;
 import java.io.FileOutputStream;
 import java.io.FileReader;
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
 import java.util.Vector;
 
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.filecache.TrackerDistributedCacheManager;
 import org.apache.hadoop.fs.FileUtil;
@@ -45,6 +49,7 @@ import org.junit.Before;
 import org.junit.Test;
 
 public class TestJvmManager {
+  private static final Log LOG = LogFactory.getLog(TestJvmManager.class);
   private static File TEST_DIR = new File(System.getProperty("test.build.data",
       "/tmp"), TestJvmManager.class.getSimpleName());
   private static int MAP_SLOTS = 1;
@@ -110,6 +115,14 @@ public class TestJvmManager {
     return script;
   }
   
+  // Create an empty shell script to run from tasks.
+  private File writeEmptyScript(String fileName) throws IOException {
+    File script = new File(TEST_DIR, fileName);
+    script.createNewFile();
+    script.setExecutable(true);
+    return script;
+  }
+  
   /**
    * Tests the jvm kill from JvmRunner and JvmManager simultaneously.
    * 
@@ -232,4 +245,152 @@ public class TestJvmManager {
   private void setThreadCaughtException() {
     threadCaughtException = true;
   }
+  
+  /**
+   * Test launchJvmAndWait overloads, and verify performance difference does
+   * not exceed TIME_DIFF_THRESHOLD
+   * 
+   * More details:
+   * - Create a Task that will simply invoke an empty script
+   * - Run the task using the default launchJvmAndWait – (RUN_JVM_COUNT) times
+   *    and measure total time 
+   * - Run the task using the overload of launchJvmAndWait which takes a list
+   *    of classpath entries – (RUN_JVM_COUNT) times, and measure total time
+   * - Assert the time difference is not more than TIME_DIFF_THRESHOLD
+   */
+  @Test
+  public void testJvmLaunchWithClasspathPerf() throws Exception {
+    
+    final int RUN_JVM_COUNT = 200;
+    final int TIME_DIFF_THRESHOLD = 10;
+    
+    String jvmTaskCmdName = Shell.WINDOWS ? "writeToFile.cmd" : "writeToFile";
+    final Vector<String> vargs = new Vector<String>(2);
+    vargs.add(writeEmptyScript(jvmTaskCmdName).getAbsolutePath());
+    
+    final File workDir = new File(TEST_DIR, "work");
+    final File stdout = new File(TEST_DIR, "stdout");
+    final File stderr = new File(TEST_DIR, "stderr");
+
+    // Ensure all files are deleted from previous tests
+    if (workDir.exists()) {
+      FileUtil.fullyDelete(workDir);
+    }
+    if (stdout.exists()) {
+      stdout.delete();
+    }
+    if (stderr.exists()) {
+      stderr.delete();
+    }
+   
+    TaskRunner taskRunner;
+    
+    // Create a class-path list
+    List<String> classPaths = new ArrayList<String>();
+    for (int clsPathElemnts = 0; clsPathElemnts < 10; ++clsPathElemnts) {
+      classPaths.add(TEST_DIR.getPath());
+      classPaths.add(workDir.getPath());
+      classPaths.add(stdout.getPath());
+      classPaths.add(stderr.getPath());
+    }
+    
+    // Get the start time before launching the tasks
+    long startTime = 0;
+    long endTime = 0;
+    long totalTimeNoJar = 0;
+    long totalTimeWithJar = 0;
+
+    // Test launching the the task without constructing classpath jar
+    for (int iNoJar = 0; iNoJar < RUN_JVM_COUNT; ++iNoJar) {
+      // Create a new task and name it using the current run count
+      taskRunner = prepareNewTask(0, iNoJar, 0);
+      
+      // Vargs are changed by the below overload of launchJvmAndWait which add
+      // the classpath as args, so we use a copy of vargs with every iteration
+      Vector<String> vargsCopy = new Vector<String>(vargs);
+      
+      // Get start time
+      startTime = System.currentTimeMillis();
+      
+      // Launch the the task without constructing classpath jar
+      taskRunner.launchJvmAndWait(null, vargsCopy, null, stdout, stderr, 
+          100, workDir);
+
+      // Get end time
+      endTime = System.currentTimeMillis();
+      
+      totalTimeNoJar = totalTimeNoJar + (endTime - startTime);
+      
+      // Clean generated files
+      workDir.delete();
+      stdout.delete();
+      stderr.delete();
+    }
+    
+    // Test launching the the task with constructing classpath jar
+    for (int iWithJar = 0; iWithJar < RUN_JVM_COUNT; ++iWithJar) {
+      // Create a new task and name it using the current run count
+      taskRunner = prepareNewTask(0, iWithJar, 1);
+      
+      // Vargs are changed by the below overload of launchJvmAndWait which add
+      // the classpath as args, so we use a copy of vargs with every iteration
+      Vector<String> vargsCopy = new Vector<String>(vargs);
+      
+      // Get start time
+      startTime = System.currentTimeMillis();
+      
+      // Launch the the task constructing classpath jar    
+      taskRunner.launchJvmAndWait(null, vargsCopy, classPaths, stdout, stderr, 
+          100, workDir);
+      
+      // Get end time
+      endTime = System.currentTimeMillis();
+      
+      totalTimeWithJar = totalTimeWithJar + (endTime - startTime);
+      
+      // Clean generated files
+      workDir.delete();
+      stdout.delete();
+      stderr.delete();
+    }
+
+    // Measure the time difference
+    double timeDiffMilli = Math.abs(totalTimeWithJar - totalTimeNoJar);
+    double diffPercentage = (timeDiffMilli / totalTimeNoJar) * 100;
+    
+    // Log results
+    LOG.info(
+        "Time taken for launchJvmAndWait without classpath (milli seconds): "
+        + totalTimeNoJar);
+    LOG.info(
+        "Time taken for launchJvmAndWait with classpath (milli seconds): "
+        + totalTimeWithJar);
+    LOG.info("Time difference is: " + diffPercentage + "%");
+    
+    // Verify difference does not exceed TIME_DIFF_THRESHOLD
+    assertTrue(diffPercentage <= TIME_DIFF_THRESHOLD);
+  }
+  
+  /**
+   * Helper function to create a new task with the given jobId, ttaskId,
+   * and attemptId
+   * Returns the TaskRunner used to launch the task
+   */
+  private TaskRunner prepareNewTask(int jobId, int ttaskId, int attemptId)
+      throws IOException{
+    JobConf taskConf = new JobConf(ttConf);
+    TaskAttemptID attemptID = new TaskAttemptID("test", jobId, true, ttaskId,
+        attemptId);
+    Task task = new MapTask(null, attemptID, 0, null, MAP_SLOTS);
+    task.setUser(user);
+    task.setConf(taskConf);
+    TaskInProgress tip = tt.new TaskInProgress(task, taskConf);
+    RunningJob rjob = new RunningJob(attemptID.getJobID());
+    TaskController taskController = new DefaultTaskController();
+    taskController.setConf(ttConf);
+    rjob.distCacheMgr = 
+      new TrackerDistributedCacheManager(ttConf, taskController).
+          newTaskDistributedCacheManager(attemptID.getJobID(), taskConf);
+    return task.createRunner(tt, tip, rjob);
+  }
 }

Modified: hadoop/common/branches/branch-1-win/src/test/testshell/ExternalMapReduce.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/test/testshell/ExternalMapReduce.java?rev=1399953&r1=1399952&r2=1399953&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/src/test/testshell/ExternalMapReduce.java (original)
+++ hadoop/common/branches/branch-1-win/src/test/testshell/ExternalMapReduce.java Fri Oct
19 02:46:16 2012
@@ -19,8 +19,13 @@
 package testshell;
 
 import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.util.Iterator;
+import java.util.jar.Attributes;
+import java.util.jar.JarInputStream;
+import java.util.jar.Manifest;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.conf.Configured;
@@ -101,6 +106,44 @@ public class ExternalMapReduce extends C
     }
   }
 
+  // Check if the given item exists in classpath
+  private static boolean itemInClasspath(String elementName) 
+    throws FileNotFoundException, IOException {
+    boolean found = false;
+    String classpath = System.getProperty("java.class.path");
+
+    // Check if the element exists in the classpath
+    if (classpath.indexOf(elementName) >= 0) {
+      found = true;
+    } else {
+      // In case of Windows, classpath is embedded in a referencing jar
+      if (Shell.WINDOWS) {
+        String[] cpFiles = classpath.split(File.pathSeparator);
+        if (cpFiles != null && cpFiles.length == 1 && 
+            cpFiles[0].endsWith(".jar")) {
+          // Search for the element in the jar manifest
+          found = itemInJarClasspath(elementName, cpFiles[0]);
+        }
+      }
+    }
+    return found;
+  }
+  
+  // Check if the given item exists as a classpath element in a jar file
+  private static boolean itemInJarClasspath(String elementName, 
+                                            String jarFileName) 
+    throws FileNotFoundException, IOException{
+    // Load the Jar manifest
+    JarInputStream jarStream = new JarInputStream(
+        new FileInputStream(jarFileName));
+    Manifest jarManifest = jarStream.getManifest();
+    String classpath = jarManifest.getMainAttributes().getValue(
+        Attributes.Name.CLASS_PATH.toString());
+    
+    // Check for the element in the classpath list
+    return (classpath.indexOf(elementName) >= 0);
+  }
+  
   public static class MapClass extends MapReduceBase 
     implements Mapper<WritableComparable, Writable,
                       WritableComparable, IntWritable> {
@@ -110,10 +153,10 @@ public class ExternalMapReduce extends C
       throws IOException {
       //check for classpath
       String classpath = System.getProperty("java.class.path");
-      if (classpath.indexOf("testjob.jar") == -1) {
+      if (!itemInClasspath("testjob.jar")) {
         throw new IOException("failed to find in the library " + classpath);
       }
-      if (classpath.indexOf("test.jar") == -1) {
+      if (!itemInClasspath("test.jar")) {
         throw new IOException("failed to find the library test.jar in" 
             + classpath);
       }



Mime
View raw message