hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From acmur...@apache.org
Subject svn commit: r634565 - in /hadoop/core/branches/branch-0.16: CHANGES.txt src/test/org/apache/hadoop/io/FileBench.java src/test/org/apache/hadoop/mapred/GenericMRLoadGenerator.java
Date Fri, 07 Mar 2008 07:07:25 GMT
Author: acmurthy
Date: Thu Mar  6 23:07:21 2008
New Revision: 634565

URL: http://svn.apache.org/viewvc?rev=634565&view=rev
Log:
Merge -r 634562:634563 from trunk to branch-0.16 to fix HADOOP-2958

Modified:
    hadoop/core/branches/branch-0.16/CHANGES.txt
    hadoop/core/branches/branch-0.16/src/test/org/apache/hadoop/io/FileBench.java
    hadoop/core/branches/branch-0.16/src/test/org/apache/hadoop/mapred/GenericMRLoadGenerator.java

Modified: hadoop/core/branches/branch-0.16/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.16/CHANGES.txt?rev=634565&r1=634564&r2=634565&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.16/CHANGES.txt (original)
+++ hadoop/core/branches/branch-0.16/CHANGES.txt Thu Mar  6 23:07:21 2008
@@ -126,6 +126,12 @@
     HADOOP-2756. NPE in DFSClient while closing DFSOutputStreams 
     under load. (rangadi)
 
+    HADOOP-2958. Fixed FileBench which broke due to HADOOP-2391 which performs
+    a check for existence of the output directory and a trivial bug in
+    GenericMRLoadGenerator where min/max word lenghts were identical since
+    they were looking at the same config variables (Chris Douglas via
+    acmurthy) 
+
 Release 0.16.0 - 2008-02-07
 
   INCOMPATIBLE CHANGES

Modified: hadoop/core/branches/branch-0.16/src/test/org/apache/hadoop/io/FileBench.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.16/src/test/org/apache/hadoop/io/FileBench.java?rev=634565&r1=634564&r2=634565&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.16/src/test/org/apache/hadoop/io/FileBench.java (original)
+++ hadoop/core/branches/branch-0.16/src/test/org/apache/hadoop/io/FileBench.java Thu Mar
 6 23:07:21 2008
@@ -111,10 +111,11 @@
     Text key = new Text();
     Text val = new Text();
 
+    final String fn = conf.get("test.filebench.name", "");
+    final Path outd = conf.getOutputPath();
     OutputFormat outf = conf.getOutputFormat();
-    Path out = conf.getOutputPath();
     RecordWriter<Text,Text> rw =
-      outf.getRecordWriter(out.getFileSystem(conf), conf, out.toString(),
+      outf.getRecordWriter(outd.getFileSystem(conf), conf, fn,
                            Reporter.NULL);
     try {
       long acc = 0L;
@@ -137,7 +138,8 @@
   @SuppressWarnings("unchecked") // InputFormat instantiation
   static long readBench(JobConf conf) throws IOException {
     InputFormat inf = conf.getInputFormat();
-    Path pin = conf.getInputPaths()[0];
+    final String fn = conf.get("test.filebench.name", "");
+    Path pin = new Path(conf.getInputPaths()[0], fn);
     FileStatus in = pin.getFileSystem(conf).getFileStatus(pin);
     RecordReader rr = inf.getRecordReader(
         new FileSplit(pin, 0, in.getLen(), conf), conf, Reporter.NULL);
@@ -173,6 +175,7 @@
       try {
         if ("-dir".equals(argv[i])) {
           root = new Path(argv[++i]).makeQualified(fs);
+          System.out.println("DIR: " + root.toString());
         } else if ("-seed".equals(argv[i])) {
           job.setLong("filebench.seed", Long.valueOf(argv[++i]));
         } else if (argv[i].startsWith("-no")) {
@@ -199,6 +202,8 @@
     fillBlocks(job);
     job.setOutputKeyClass(Text.class);
     job.setOutputValueClass(Text.class);
+    job.setInputPath(root);
+    job.setOutputPath(root);
 
     if (null == cc) cc = EnumSet.allOf(CCodec.class);
     if (null == ct) ct = EnumSet.allOf(CType.class);
@@ -217,7 +222,7 @@
                 typ.name().toUpperCase();
               typ.configure(job);
               System.out.print(rwop.name().toUpperCase() + " " + fn + ": ");
-              System.out.println(rwop.exec(new Path(root, fn), job) / 1000 +
+              System.out.println(rwop.exec(fn, job) / 1000 +
                   " seconds");
             }
           } else {
@@ -226,10 +231,10 @@
               cod.name().toUpperCase();
             Path p = new Path(root, fn);
             if (rwop == RW.r && !fs.exists(p)) {
-              p = new Path(root, fn + cod.getExt());
+              fn += cod.getExt();
             }
             System.out.print(rwop.name().toUpperCase() + " " + fn + ": ");
-            System.out.println(rwop.exec(p, job) / 1000 +
+            System.out.println(rwop.exec(fn, job) / 1000 +
                 " seconds");
           }
         }
@@ -287,20 +292,20 @@
   }
   enum RW {
     w() {
-      public long exec(Path p, JobConf job) throws IOException {
-        job.setOutputPath(p);
+      public long exec(String fn, JobConf job) throws IOException {
+        job.set("test.filebench.name", fn);
         return writeBench(job);
       }
     },
 
     r() {
-      public long exec(Path p, JobConf job) throws IOException {
-        job.setInputPath(p);
+      public long exec(String fn, JobConf job) throws IOException {
+        job.set("test.filebench.name", fn);
         return readBench(job);
       }
     };
 
-    public abstract long exec(Path p, JobConf job) throws IOException;
+    public abstract long exec(String fn, JobConf job) throws IOException;
   }
   static Map<Class<? extends Enum>, Map<String,? extends Enum>> fullmap
     = new HashMap<Class<? extends Enum>, Map<String,? extends Enum>>();

Modified: hadoop/core/branches/branch-0.16/src/test/org/apache/hadoop/mapred/GenericMRLoadGenerator.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.16/src/test/org/apache/hadoop/mapred/GenericMRLoadGenerator.java?rev=634565&r1=634564&r2=634565&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.16/src/test/org/apache/hadoop/mapred/GenericMRLoadGenerator.java
(original)
+++ hadoop/core/branches/branch-0.16/src/test/org/apache/hadoop/mapred/GenericMRLoadGenerator.java
Thu Mar  6 23:07:21 2008
@@ -269,9 +269,9 @@
       bytesToWrite = job.getLong("test.randomtextwrite.bytes_per_map",
                                     1*1024*1024*1024);
       keymin = job.getInt("test.randomtextwrite.min_words_key", 5);
-      keymax = job.getInt("test.randomtextwrite.min_words_key", 10);
+      keymax = job.getInt("test.randomtextwrite.max_words_key", 10);
       valmin = job.getInt("test.randomtextwrite.min_words_value", 5);
-      valmax = job.getInt("test.randomtextwrite.min_words_value", 10);
+      valmax = job.getInt("test.randomtextwrite.max_words_value", 10);
     }
 
     public void map(Text key, Text val, OutputCollector<Text,Text> output,



Mime
View raw message