hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ha...@apache.org
Subject hadoop git commit: MAPREDUCE-2398. MRBench: setting the baseDir parameter has no effect. Contributed by Michael Noll and Wilfred Spiegelenburg.
Date Thu, 21 Apr 2016 12:26:39 GMT
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 75cf23835 -> 2d7e4a822


MAPREDUCE-2398. MRBench: setting the baseDir parameter has no effect. Contributed by Michael
Noll and Wilfred Spiegelenburg.

(cherry picked from commit 7da5847cf10bc61f2131d578f16ee37cb32c4639)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2d7e4a82
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2d7e4a82
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2d7e4a82

Branch: refs/heads/branch-2
Commit: 2d7e4a8228e6f42f5c20506f095fe43b2c8f53d8
Parents: 75cf238
Author: Harsh J <harsh@cloudera.com>
Authored: Thu Apr 21 17:33:22 2016 +0530
Committer: Harsh J <harsh@cloudera.com>
Committed: Thu Apr 21 17:54:24 2016 +0530

----------------------------------------------------------------------
 .../java/org/apache/hadoop/mapred/MRBench.java    | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d7e4a82/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MRBench.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MRBench.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MRBench.java
index bb287cf..0516445 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MRBench.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MRBench.java
@@ -41,10 +41,13 @@ import org.apache.hadoop.util.ToolRunner;
 public class MRBench extends Configured implements Tool{
   
   private static final Log LOG = LogFactory.getLog(MRBench.class);
+  private static final String DEFAULT_INPUT_SUB = "mr_input";
+  private static final String DEFAULT_OUTPUT_SUB = "mr_output";
+
   private static Path BASE_DIR =
     new Path(System.getProperty("test.build.data","/benchmarks/MRBench"));
-  private static Path INPUT_DIR = new Path(BASE_DIR, "mr_input");
-  private static Path OUTPUT_DIR = new Path(BASE_DIR, "mr_output");
+  private static Path INPUT_DIR = new Path(BASE_DIR, DEFAULT_INPUT_SUB);
+  private static Path OUTPUT_DIR = new Path(BASE_DIR, DEFAULT_OUTPUT_SUB);
   
   public static enum Order {RANDOM, ASCENDING, DESCENDING}; 
   
@@ -243,6 +246,8 @@ public class MRBench extends Configured implements Tool{
         numRuns = Integer.parseInt(args[++i]);
       } else if (args[i].equals("-baseDir")) {
         BASE_DIR = new Path(args[++i]);
+        INPUT_DIR = new Path(BASE_DIR, DEFAULT_INPUT_SUB);
+        OUTPUT_DIR = new Path(BASE_DIR, DEFAULT_OUTPUT_SUB);
       } else if (args[i].equals("-maps")) {
         numMaps = Integer.parseInt(args[++i]);
       } else if (args[i].equals("-reduces")) {
@@ -283,14 +288,15 @@ public class MRBench extends Configured implements Tool{
     Path inputFile = new Path(INPUT_DIR, "input_" + (new Random()).nextInt() + ".txt");
     generateTextFile(fs, inputFile, inputLines, inputSortOrder);
 
-    // setup test output directory
-    fs.mkdirs(BASE_DIR); 
     ArrayList<Long> execTimes = new ArrayList<Long>();
     try {
       execTimes = runJobInSequence(jobConf, numRuns);
     } finally {
-      // delete output -- should we really do this?
-      fs.delete(BASE_DIR, true);
+      // delete all generated data -- should we really do this?
+      // we don't know how much of the path was created for the run but this
+      // cleans up as much as we can
+      fs.delete(OUTPUT_DIR, true);
+      fs.delete(INPUT_DIR, true);
     }
     
     if (verbose) {


Mime
View raw message