hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From omal...@apache.org
Subject svn commit: r1077099 - in /hadoop/common/branches/branch-0.20-security-patches: ./ src/contrib/ src/contrib/streaming/src/test/org/apache/hadoop/streaming/ src/mapred/org/apache/hadoop/mapred/ src/test/org/apache/hadoop/mapred/ src/test/org/apache/hado...
Date Fri, 04 Mar 2011 03:40:26 GMT
Author: omalley
Date: Fri Mar  4 03:40:25 2011
New Revision: 1077099

URL: http://svn.apache.org/viewvc?rev=1077099&view=rev
Log:
commit 32ab54df050f8de4dee11e9b9c125a67a624f971
Author: Hemanth Yamijala <yhemanth@yahoo-inc.com>
Date:   Mon Jan 11 19:38:08 2010 +0530

    Reverting patch https://issues.apache.org/jira/secure/attachment/12426091/hadoop-5771-ydist.patch
for HADOOP-5771
    
    +++ b/YAHOO-CHANGES.txt
    +    HADOOP-5771. Implements unit tests for LinuxTaskController.
    +    (Sreekanth Ramakrishnan and Vinod Kumar Vavilapalli via yhemanth)
    +

Removed:
    hadoop/common/branches/branch-0.20-security-patches/src/contrib/streaming/src/test/org/apache/hadoop/streaming/TestStreamingAsDifferentUser.java
    hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/ClusterWithLinuxTaskController.java
    hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/TestJobExecutionAsDifferentUser.java
    hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/pipes/TestPipesAsDifferentUser.java
Modified:
    hadoop/common/branches/branch-0.20-security-patches/build.xml
    hadoop/common/branches/branch-0.20-security-patches/src/contrib/build-contrib.xml
    hadoop/common/branches/branch-0.20-security-patches/src/contrib/streaming/src/test/org/apache/hadoop/streaming/TrApp.java
    hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/LinuxTaskController.java
    hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/TestMiniMRChildTask.java
    hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/pipes/TestPipes.java

Modified: hadoop/common/branches/branch-0.20-security-patches/build.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/build.xml?rev=1077099&r1=1077098&r2=1077099&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/build.xml (original)
+++ hadoop/common/branches/branch-0.20-security-patches/build.xml Fri Mar  4 03:40:25 2011
@@ -740,8 +740,6 @@
       <sysproperty key="test.debug.data" value="${test.debug.data}"/>
       <sysproperty key="hadoop.log.dir" value="${test.log.dir}"/>
       <sysproperty key="test.src.dir" value="${test.src.dir}"/>
-    	<sysproperty key="taskcontroller-path" value="${taskcontroller-path}"/>
-    	<sysproperty key="taskcontroller-user" value="${taskcontroller-user}"/>
       <sysproperty key="test.build.extraconf" value="${test.build.extraconf}" />
       <sysproperty key="hadoop.policy.file" value="hadoop-policy.xml"/>
       <sysproperty key="java.library.path"

Modified: hadoop/common/branches/branch-0.20-security-patches/src/contrib/build-contrib.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/contrib/build-contrib.xml?rev=1077099&r1=1077098&r2=1077099&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/contrib/build-contrib.xml (original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/contrib/build-contrib.xml Fri
Mar  4 03:40:25 2011
@@ -233,8 +233,6 @@
       <sysproperty key="fs.default.name" value="${fs.default.name}"/>
       <sysproperty key="hadoop.test.localoutputfile" value="${hadoop.test.localoutputfile}"/>
       <sysproperty key="hadoop.log.dir" value="${hadoop.log.dir}"/> 
-      <sysproperty key="taskcontroller-path" value="${taskcontroller-path}"/>
-      <sysproperty key="taskcontroller-user" value="${taskcontroller-user}"/>
       <classpath refid="test.classpath"/>
       <formatter type="${test.junit.output.format}" />
       <batchtest todir="${build.test}" unless="testcase">

Modified: hadoop/common/branches/branch-0.20-security-patches/src/contrib/streaming/src/test/org/apache/hadoop/streaming/TrApp.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/contrib/streaming/src/test/org/apache/hadoop/streaming/TrApp.java?rev=1077099&r1=1077098&r2=1077099&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/contrib/streaming/src/test/org/apache/hadoop/streaming/TrApp.java
(original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/contrib/streaming/src/test/org/apache/hadoop/streaming/TrApp.java
Fri Mar  4 03:40:25 2011
@@ -41,6 +41,8 @@ public class TrApp
     // test that some JobConf properties are exposed as expected     
     // Note the dots translated to underscore: 
     // property names have been escaped in PipeMapRed.safeEnvVarName()
+    expect("mapred_job_tracker", "local");
+    //expect("mapred_local_dir", "build/test/mapred/local");
     expectDefined("mapred_local_dir");
     expect("mapred_output_format_class", "org.apache.hadoop.mapred.TextOutputFormat");
     expect("mapred_output_key_class", "org.apache.hadoop.io.Text");
@@ -50,6 +52,7 @@ public class TrApp
     expectDefined("mapred_task_id");
 
     expectDefined("map_input_file");
+    expect("map_input_start", "0");
     expectDefined("map_input_length");
 
     expectDefined("io_sort_factor");

Modified: hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/LinuxTaskController.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/LinuxTaskController.java?rev=1077099&r1=1077098&r2=1077099&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/LinuxTaskController.java
(original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/LinuxTaskController.java
Fri Mar  4 03:40:25 2011
@@ -408,7 +408,7 @@ class LinuxTaskController extends TaskCo
       File workDir, Map<String, String> env) 
       throws IOException {
     String[] taskControllerCmd = new String[3 + cmdArgs.size()];
-    taskControllerCmd[0] = getTaskControllerExecutablePath();
+    taskControllerCmd[0] = taskControllerExe;
     taskControllerCmd[1] = userName;
     taskControllerCmd[2] = String.valueOf(command.ordinal());
     int i = 3;
@@ -475,10 +475,7 @@ class LinuxTaskController extends TaskCo
       }
     }
   }
-
-  protected String getTaskControllerExecutablePath() {
-    return taskControllerExe;
-  }  
+  
 
   /**
    * Sets up the permissions of the following directories:

Modified: hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/TestMiniMRChildTask.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/TestMiniMRChildTask.java?rev=1077099&r1=1077098&r2=1077099&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/TestMiniMRChildTask.java
(original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/TestMiniMRChildTask.java
Fri Mar  4 03:40:25 2011
@@ -415,23 +415,4 @@ public class TestMiniMRChildTask extends
     RunningJob job = JobClient.runJob(conf);
     assertTrue("The environment checker job failed.", job.isSuccessful());
   }
-  
-  void runTestTaskEnv(JobConf conf, Path inDir, Path outDir) throws IOException {
-    String input = "The input";
-    configure(conf, inDir, outDir, input, EnvCheckMapper.class, 
-        IdentityReducer.class);
-    // test 
-    //  - new SET of new var (MY_PATH)
-    //  - set of old var (HOME)
-    //  - append to an old var from modified env (LD_LIBRARY_PATH)
-    //  - append to an old var from tt's env (PATH)
-    //  - append to a new var (NEW_PATH)
-    conf.set("mapred.child.env", 
-             "MY_PATH=/tmp,HOME=/tmp,LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/tmp,"
-             + "PATH=$PATH:/tmp,NEW_PATH=$NEW_PATH:/tmp");
-    conf.set("path", System.getenv("PATH"));
-    RunningJob job = JobClient.runJob(conf);
-    assertTrue("The environment checker job failed.", job.isSuccessful());
-  }
-  
 }

Modified: hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/pipes/TestPipes.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/pipes/TestPipes.java?rev=1077099&r1=1077098&r2=1077099&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/pipes/TestPipes.java
(original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/pipes/TestPipes.java
Fri Mar  4 03:40:25 2011
@@ -32,8 +32,6 @@ import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.FileUtil;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.hadoop.fs.permission.FsAction;
-import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.mapred.Counters;
 import org.apache.hadoop.mapred.FileInputFormat;
 import org.apache.hadoop.mapred.FileOutputFormat;
@@ -49,18 +47,7 @@ import org.apache.hadoop.util.ToolRunner
 public class TestPipes extends TestCase {
   private static final Log LOG =
     LogFactory.getLog(TestPipes.class.getName());
-  
-  private static Path cppExamples = 
-    new Path(System.getProperty("install.c++.examples"));
-  static Path wordCountSimple = 
-    new Path(cppExamples, "bin/wordcount-simple");
-  static Path wordCountPart = 
-    new Path(cppExamples, "bin/wordcount-part");
-  static Path wordCountNoPipes = 
-    new Path(cppExamples,"bin/wordcount-nopipe");
-  
-  static Path nonPipedOutDir;
-  
+
   static void cleanup(FileSystem fs, Path p) throws IOException {
     fs.delete(p, true);
     assertFalse("output not cleaned up", fs.exists(p));
@@ -73,6 +60,7 @@ public class TestPipes extends TestCase 
     }
     MiniDFSCluster dfs = null;
     MiniMRCluster mr = null;
+    Path cppExamples = new Path(System.getProperty("install.c++.examples"));
     Path inputPath = new Path("/testing/in");
     Path outputPath = new Path("/testing/out");
     try {
@@ -81,15 +69,17 @@ public class TestPipes extends TestCase 
       dfs = new MiniDFSCluster(conf, numSlaves, true, null);
       mr = new MiniMRCluster(numSlaves, dfs.getFileSystem().getName(), 1);
       writeInputFile(dfs.getFileSystem(), inputPath);
-      runProgram(mr, dfs, wordCountSimple, 
-                 inputPath, outputPath, 3, 2, twoSplitOutput, null);
+      runProgram(mr, dfs, new Path(cppExamples, "bin/wordcount-simple"), 
+                 inputPath, outputPath, 3, 2, twoSplitOutput);
       cleanup(dfs.getFileSystem(), outputPath);
-      runProgram(mr, dfs, wordCountSimple, 
-                 inputPath, outputPath, 3, 0, noSortOutput, null);
+
+      runProgram(mr, dfs, new Path(cppExamples, "bin/wordcount-simple"), 
+                 inputPath, outputPath, 3, 0, noSortOutput);
       cleanup(dfs.getFileSystem(), outputPath);
-      runProgram(mr, dfs, wordCountPart,
-                 inputPath, outputPath, 3, 2, fixedPartitionOutput, null);
-      runNonPipedProgram(mr, dfs, wordCountNoPipes, null);
+
+      runProgram(mr, dfs, new Path(cppExamples, "bin/wordcount-part"),
+                 inputPath, outputPath, 3, 2, fixedPartitionOutput);
+      runNonPipedProgram(mr, dfs, new Path(cppExamples,"bin/wordcount-nopipe"));
       mr.waitUntilIdle();
     } finally {
       mr.shutdown();
@@ -97,7 +87,6 @@ public class TestPipes extends TestCase 
     }
   }
 
-
   final static String[] twoSplitOutput = new String[] {
     "`and\t1\na\t1\nand\t1\nbeginning\t1\nbook\t1\nbut\t1\nby\t1\n" +
     "conversation?'\t1\ndo:\t1\nhad\t2\nhaving\t1\nher\t2\nin\t1\nit\t1\n"+
@@ -135,7 +124,7 @@ public class TestPipes extends TestCase 
     ""                                                   
   };
   
-  static void writeInputFile(FileSystem fs, Path dir) throws IOException {
+  private void writeInputFile(FileSystem fs, Path dir) throws IOException {
     DataOutputStream out = fs.create(new Path(dir, "part0"));
     out.writeBytes("Alice was beginning to get very tired of sitting by her\n");
     out.writeBytes("sister on the bank, and of having nothing to do: once\n");
@@ -146,18 +135,12 @@ public class TestPipes extends TestCase 
     out.close();
   }
 
-  static void runProgram(MiniMRCluster mr, MiniDFSCluster dfs, 
+  private void runProgram(MiniMRCluster mr, MiniDFSCluster dfs, 
                           Path program, Path inputPath, Path outputPath,
-                          int numMaps, int numReduces, String[] expectedResults,
-                          JobConf conf
+                          int numMaps, int numReduces, String[] expectedResults
                          ) throws IOException {
     Path wordExec = new Path("/testing/bin/application");
-    JobConf job = null;
-    if(conf == null) {
-      job = mr.createJobConf();
-    }else {
-      job = new JobConf(conf);
-    } 
+    JobConf job = mr.createJobConf();
     job.setNumMapTasks(numMaps);
     job.setNumReduceTasks(numReduces);
     {
@@ -216,21 +199,15 @@ public class TestPipes extends TestCase 
    * @param program the program to run
    * @throws IOException
    */
-  static void runNonPipedProgram(MiniMRCluster mr, MiniDFSCluster dfs,
-                                  Path program, JobConf conf) throws IOException {
-    JobConf job;
-    if(conf == null) {
-      job = mr.createJobConf();
-    }else {
-      job = new JobConf(conf);
-    }
-    
+  private void runNonPipedProgram(MiniMRCluster mr, MiniDFSCluster dfs,
+                                  Path program) throws IOException {
+    JobConf job = mr.createJobConf();
     job.setInputFormat(WordCountInputFormat.class);
     FileSystem local = FileSystem.getLocal(job);
     Path testDir = new Path("file:" + System.getProperty("test.build.data"), 
                             "pipes");
     Path inDir = new Path(testDir, "input");
-    nonPipedOutDir = new Path(testDir, "output");
+    Path outDir = new Path(testDir, "output");
     Path wordExec = new Path("/testing/bin/application");
     Path jobXml = new Path(testDir, "job.xml");
     {
@@ -251,21 +228,21 @@ public class TestPipes extends TestCase 
     out.writeBytes("hall silly cats drink java\n");
     out.writeBytes("all dogs bow wow\n");
     out.writeBytes("hello drink java\n");
-    local.delete(nonPipedOutDir, true);
-    local.mkdirs(nonPipedOutDir, new FsPermission(FsAction.ALL, FsAction.ALL,
-        FsAction.ALL));
     out.close();
+    local.delete(outDir, true);
+    local.mkdirs(outDir);
     out = local.create(jobXml);
     job.writeXml(out);
     out.close();
-    System.err.println("About to run: Submitter -conf " + jobXml + " -input "
-        + inDir + " -output " + nonPipedOutDir + " -program "
-        + dfs.getFileSystem().makeQualified(wordExec));
+    System.err.println("About to run: Submitter -conf " + jobXml + 
+                       " -input " + inDir + " -output " + outDir + 
+                       " -program " + 
+                       dfs.getFileSystem().makeQualified(wordExec));
     try {
       int ret = ToolRunner.run(new Submitter(),
                                new String[]{"-conf", jobXml.toString(),
                                   "-input", inDir.toString(),
-                                  "-output", nonPipedOutDir.toString(),
+                                  "-output", outDir.toString(),
                                   "-program", 
                         dfs.getFileSystem().makeQualified(wordExec).toString(),
                                   "-reduces", "2"});



Mime
View raw message