hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cutt...@apache.org
Subject svn commit: r430838 - in /lucene/hadoop/trunk: ./ src/contrib/smallJobsBenchmark/ src/contrib/smallJobsBenchmark/bin/ src/contrib/smallJobsBenchmark/src/java/org/apache/hadoop/benchmarks/mapred/
Date Fri, 11 Aug 2006 18:04:59 GMT
Author: cutting
Date: Fri Aug 11 11:04:58 2006
New Revision: 430838

URL: http://svn.apache.org/viewvc?rev=430838&view=rev
Log:
HADOOP-434.  Change smallJobsBenchmark to user standard Hadoop scripts.  Contributed by Sanjay
Dahiya.

Modified:
    lucene/hadoop/trunk/CHANGES.txt
    lucene/hadoop/trunk/src/contrib/smallJobsBenchmark/Readme.txt
    lucene/hadoop/trunk/src/contrib/smallJobsBenchmark/bin/run.sh
    lucene/hadoop/trunk/src/contrib/smallJobsBenchmark/build.xml
    lucene/hadoop/trunk/src/contrib/smallJobsBenchmark/src/java/org/apache/hadoop/benchmarks/mapred/MultiJobRunner.java

Modified: lucene/hadoop/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/CHANGES.txt?rev=430838&r1=430837&r2=430838&view=diff
==============================================================================
--- lucene/hadoop/trunk/CHANGES.txt (original)
+++ lucene/hadoop/trunk/CHANGES.txt Fri Aug 11 11:04:58 2006
@@ -19,6 +19,9 @@
  4. HADOOP-324.  Fix datanode to not exit when a disk is full, but
     rather simply to fail writes.  (Wendy Chien via cutting)
 
+ 5. HADOOP-434.  Change smallJobsBenchmark to use standard Hadoop
+    scripts.  (Sanjay Dahiya via cutting)
+
 
 Release 0.5.0 - 2006-08-04
 

Modified: lucene/hadoop/trunk/src/contrib/smallJobsBenchmark/Readme.txt
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/smallJobsBenchmark/Readme.txt?rev=430838&r1=430837&r2=430838&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/contrib/smallJobsBenchmark/Readme.txt (original)
+++ lucene/hadoop/trunk/src/contrib/smallJobsBenchmark/Readme.txt Fri Aug 11 11:04:58 2006
@@ -6,6 +6,9 @@
 $ ant deploy
 
 Running the benchmark
+$ $HADOOP_HOME/bin/hadoop jar <path to MRBenchmark.jar> smallJobsBenchmark -inputLines
<numLines> -maps <numMaps> -reduces <numReduces> -jar <path to MRBenchmark.jar>
+
+For running the benchmark multiple times with different input sizes, maps, reduces following
helper scripts can be used. 
 $ cd build/contrib/smallJobsBenchmark
 $ bin/run.sh
 

Modified: lucene/hadoop/trunk/src/contrib/smallJobsBenchmark/bin/run.sh
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/smallJobsBenchmark/bin/run.sh?rev=430838&r1=430837&r2=430838&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/contrib/smallJobsBenchmark/bin/run.sh (original)
+++ lucene/hadoop/trunk/src/contrib/smallJobsBenchmark/bin/run.sh Fri Aug 11 11:04:58 2006
@@ -12,8 +12,6 @@
   exit 1;
 fi
 
-export CLASSPATH=$HADOOP_HOME/conf:$HADOOP_HOME/build/classes:$HADOOP_HOME/build:$HADOOP_HOME/build/test/classes:$HADOOP_HOME/hadoop-*.jar:$HADOOP_HOME/lib/commons-cli-2.0-SNAPSHOT.jar:$HADOOP_HOME/lib/commons-logging-1.0.4.jar:$HADOOP_HOME/lib/commons-logging-api-1.0.4.jar:$HADOOP_HOME/lib/jetty-5.1.4.jar:$HADOOP_HOME/lib/junit-3.8.1.jar:$HADOOP_HOME/lib/log4j-1.2.13.jar:$HADOOP_HOME/lib/lucene-core-1.9.1.jar:$HADOOP_HOME/lib/servlet-api.jar:$HADOOP_HOME/lib/jetty-ext/ant.jar:$HADOOP_HOME/lib/jetty-ext/commons-el.jar:$HADOOP_HOME/lib/jetty-ext/jasper-compiler.jar:$HADOOP_HOME/lib/jetty-ext/jasper-runtime.jar:$HADOOP_HOME/lib/jetty-ext/jsp-api.jar
-
 mkdir -p logs;
 
 export TIMES=2
@@ -26,7 +24,7 @@
 	do 
 	for reduces in 1 18
 	do
-$JAVA_HOME/bin/java -classpath $CLASSPATH:./classes org.apache.hadoop.benchmarks.mapred.MultiJobRunner
-inputLines ${dataLines} -output /hadoop/mapred/MROutput -jar MRBenchmark.jar -times ${TIMES}
-workDir /hadoop/mapred/work -maps ${maps} -reduces ${reduces} -inputType ascending  -ignoreOutput
 2>&1 | tee logs/benchmark_${dataLines}_${maps}_${reduces}.log
+$HADOOP_HOME/bin/hadoop jar $HADOOP_HOME/../contrib/smallJobsBenchmark/MRBenchmark.jar smallJobsBenchmark
-inputLines ${dataLines} -output /hadoop/mapred/MROutput -jar $HADOOP_HOME/../contrib/smallJobsBenchmark/MRBenchmark.jar
-times ${TIMES} -workDir /hadoop/mapred/work -maps ${maps} -reduces ${reduces} -inputType
ascending  -ignoreOutput  2>&1 | tee logs/benchmark_${dataLines}_${maps}_${reduces}.log
 
 	done
 	done

Modified: lucene/hadoop/trunk/src/contrib/smallJobsBenchmark/build.xml
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/smallJobsBenchmark/build.xml?rev=430838&r1=430837&r2=430838&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/contrib/smallJobsBenchmark/build.xml (original)
+++ lucene/hadoop/trunk/src/contrib/smallJobsBenchmark/build.xml Fri Aug 11 11:04:58 2006
@@ -1,6 +1,6 @@
 <?xml version="1.0"?>
 
-<project name="smallJobsBenchmark" default="jar">
+<project name="smallJobsBenchmark" default="deploy">
 
   <import file="../build-contrib.xml"/>
   
@@ -12,6 +12,10 @@
     <jar
       jarfile="${build.dir}/${benchmarkJarName}"
       basedir="${build.classes}">
+    <manifest>
+       <attribute name="Main-Class" 
+                  value="org/apache/hadoop/benchmarks/mapred/BenchmarkRunner"/>
+     </manifest>
     </jar>
   </target>
   

Modified: lucene/hadoop/trunk/src/contrib/smallJobsBenchmark/src/java/org/apache/hadoop/benchmarks/mapred/MultiJobRunner.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/smallJobsBenchmark/src/java/org/apache/hadoop/benchmarks/mapred/MultiJobRunner.java?rev=430838&r1=430837&r2=430838&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/contrib/smallJobsBenchmark/src/java/org/apache/hadoop/benchmarks/mapred/MultiJobRunner.java
(original)
+++ lucene/hadoop/trunk/src/contrib/smallJobsBenchmark/src/java/org/apache/hadoop/benchmarks/mapred/MultiJobRunner.java
Fri Aug 11 11:04:58 2006
@@ -26,10 +26,10 @@
  */
 public class MultiJobRunner {
   
-  private String jarFile = "MRBenchmark.jar" ;
+  private String jarFile = null ; // "MRBenchmark.jar" ;
   private String input ; 
   private String output ; 
-  private int numJobs = 2000 ; // default value
+  private int numJobs = 1 ; // default value
   private static final Log LOG = LogFactory.getLog(MultiJobRunner.class);
   private int numMaps = 2; 
   private int numReduces = 1;
@@ -130,7 +130,9 @@
     
     job.setOutputPath(new Path(output));
     
-    job.setJar(jarFile);
+    if( null != jarFile ){
+      job.setJar(jarFile);
+    }
     job.setMapperClass(BenchmarkMapper.class);
     job.setReducerClass(BenchmarkReducer.class);
     
@@ -338,8 +340,8 @@
     }
     
     String output = "";
-    String jarFile = "MRBenchmark.jar" ; 
-    int numJobs = 0 ; 
+    String jarFile = null; //"MRBenchmark.jar" ; 
+    int numJobs = 1 ; 
     int numMaps = 2; 
     int numReduces = 1 ; 
     int dataLines = 1 ; 



Mime
View raw message