hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cdoug...@apache.org
Subject svn commit: r630058 - in /hadoop/core/trunk: ./ src/test/gridmix/ src/test/gridmix/maxent/ src/test/gridmix/submissionScripts/
Date Fri, 22 Feb 2008 00:33:58 GMT
Author: cdouglas
Date: Thu Feb 21 16:33:56 2008
New Revision: 630058

URL: http://svn.apache.org/viewvc?rev=630058&view=rev
Log:
HADOOP-2852. Update gridmix benchmark to avoid an artifically long tail.


Modified:
    hadoop/core/trunk/CHANGES.txt
    hadoop/core/trunk/src/test/gridmix/gridmix-env
    hadoop/core/trunk/src/test/gridmix/maxent/maxent.large
    hadoop/core/trunk/src/test/gridmix/submissionScripts/allThroughHod
    hadoop/core/trunk/src/test/gridmix/submissionScripts/allToSameCluster
    hadoop/core/trunk/src/test/gridmix/submissionScripts/maxentHod
    hadoop/core/trunk/src/test/gridmix/submissionScripts/sleep_if_too_busy

Modified: hadoop/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=630058&r1=630057&r2=630058&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Thu Feb 21 16:33:56 2008
@@ -137,6 +137,9 @@
     HADOOP-2769.  TestNNThroughputBnechmark should not use a fixed port for
     the namenode http port. (omalley)
 
+    HADOOP-2852. Update gridmix benchmark to avoid an artifically long tail.
+    (cdouglas)
+
 Release 0.16.0 - 2008-02-07
 
   INCOMPATIBLE CHANGES

Modified: hadoop/core/trunk/src/test/gridmix/gridmix-env
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/gridmix/gridmix-env?rev=630058&r1=630057&r2=630058&view=diff
==============================================================================
--- hadoop/core/trunk/src/test/gridmix/gridmix-env (original)
+++ hadoop/core/trunk/src/test/gridmix/gridmix-env Thu Feb 21 16:33:56 2008
@@ -30,6 +30,7 @@
 export FIXCOMPTEXT=${GRID_MIX_DATA}/EntropySimulationCompressed
 
 ## Job sizing
+export NUM_OF_LARGE_JOBS_FOR_ENTROPY_CLASS=5
 export NUM_OF_LARGE_JOBS_PER_CLASS=3
 export NUM_OF_MEDIUM_JOBS_PER_CLASS=20
 export NUM_OF_SMALL_JOBS_PER_CLASS=40
@@ -44,7 +45,7 @@
 ## Hod
 #export HOD_OPTIONS="--ringmaster.hadoop-tar-ball=/path/to/hadoop-0.15.0-dev.tar.gz"
 #export HOD_CONFIG=
-#export ALL_HOD_OPTIONS="$HOD_OPTIONS -c ${HOD_CONFIG}"
+#export ALL_HOD_OPTIONS="-c ${HOD_CONFIG} ${HOD_OPTIONS}"
 #export SMALL_JOB_HOD_OPTIONS="$ALL_HOD_OPTIONS -m 5"
 #export MEDIUM_JOB_HOD_OPTIONS="$ALL_HOD_OPTIONS -m 50"
 #export LARGE_JOB_HOD_OPTIONS="$ALL_HOD_OPTIONS -m 100"

Modified: hadoop/core/trunk/src/test/gridmix/maxent/maxent.large
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/gridmix/maxent/maxent.large?rev=630058&r1=630057&r2=630058&view=diff
==============================================================================
--- hadoop/core/trunk/src/test/gridmix/maxent/maxent.large (original)
+++ hadoop/core/trunk/src/test/gridmix/maxent/maxent.large Thu Feb 21 16:33:56 2008
@@ -13,7 +13,7 @@
 
 ${HADOOP_HOME}/bin/hadoop jar $APP_JAR loadgen -keepmap 50 -keepred 100 -inFormatIndirect
org.apache.hadoop.mapred.TextInputFormat -outFormat org.apache.hadoop.mapred.TextOutputFormat
-outKey org.apache.hadoop.io.LongWritable -outValue org.apache.hadoop.io.Text -indir $INDIR
-outdir $OUTDIR.1 -r $NUM_OF_REDUCERS
 
-ITER=11
+ITER=7
 for ((i=1; i<$ITER; ++i))
 do
   ${HADOOP_HOME}/bin/hadoop jar $APP_JAR loadgen -keepmap 50 -keepred 100 -inFormatIndirect
org.apache.hadoop.mapred.TextInputFormat -outFormat org.apache.hadoop.mapred.TextOutputFormat
-outKey org.apache.hadoop.io.LongWritable -outValue org.apache.hadoop.io.Text -indir $INDIR
-indir $OUTDIR.$i -outdir $OUTDIR.$(($i+1)) -r $NUM_OF_REDUCERS

Modified: hadoop/core/trunk/src/test/gridmix/submissionScripts/allThroughHod
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/gridmix/submissionScripts/allThroughHod?rev=630058&r1=630057&r2=630058&view=diff
==============================================================================
--- hadoop/core/trunk/src/test/gridmix/submissionScripts/allThroughHod (original)
+++ hadoop/core/trunk/src/test/gridmix/submissionScripts/allThroughHod Thu Feb 21 16:33:56
2008
@@ -4,10 +4,10 @@
 GRID_DIR=`cd "$GRID_DIR"; pwd`
 source $GRID_DIR/../gridmix-env
 
+$GRID_MIX_HOME/submissionScripts/maxentHod 2>&1 >  maxentHod.out & 
 $GRID_MIX_HOME/submissionScripts/textSortHod 2>&1 >  textSortHod.out &
 $GRID_MIX_HOME/submissionScripts/monsterQueriesHod 2>&1 > monsterQueriesHod.out
&
 $GRID_MIX_HOME/submissionScripts/webdataScanHod 2>&1 > webdataScanHod.out &
 $GRID_MIX_HOME/submissionScripts/webdataSortHod 2>&1 > webdataSortHod.out &
-$GRID_MIX_HOME/submissionScripts/maxentHod 2>&1 >  maxentHod.out & 
 
 

Modified: hadoop/core/trunk/src/test/gridmix/submissionScripts/allToSameCluster
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/gridmix/submissionScripts/allToSameCluster?rev=630058&r1=630057&r2=630058&view=diff
==============================================================================
--- hadoop/core/trunk/src/test/gridmix/submissionScripts/allToSameCluster (original)
+++ hadoop/core/trunk/src/test/gridmix/submissionScripts/allToSameCluster Thu Feb 21 16:33:56
2008
@@ -4,6 +4,8 @@
 GRID_DIR=`cd "$GRID_DIR"; pwd`
 source $GRID_DIR/../gridmix-env
 
+$GRID_MIX_HOME/submissionScripts/maxentToSameCluster 2>&1 > maxentToSameCluster.out
&
+sleep 20
 $GRID_MIX_HOME/submissionScripts/textSortToSameCluster 2>&1 > textSortToSameCluster.out
 &
 sleep 20
 $GRID_MIX_HOME/submissionScripts/monsterQueriesToSameCluster 2>&1 > monsterQueriesToSameCluster.out
&
@@ -11,6 +13,4 @@
 $GRID_MIX_HOME/submissionScripts/webdataScanToSameCluster 2>&1 > webdataScanToSameCluster.out
&
 sleep 20
 $GRID_MIX_HOME/submissionScripts/webdataSortToSameCluster  2>&1 > webdataSortToSameCluster.out
&
-sleep 20
-$GRID_MIX_HOME/submissionScripts/maxentToSameCluster 2>&1 > maxentToSameCluster.out
&
 

Modified: hadoop/core/trunk/src/test/gridmix/submissionScripts/maxentHod
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/gridmix/submissionScripts/maxentHod?rev=630058&r1=630057&r2=630058&view=diff
==============================================================================
--- hadoop/core/trunk/src/test/gridmix/submissionScripts/maxentHod (original)
+++ hadoop/core/trunk/src/test/gridmix/submissionScripts/maxentHod Thu Feb 21 16:33:56 2008
@@ -3,8 +3,7 @@
 GRID_DIR=`dirname "$0"`
 GRID_DIR=`cd "$GRID_DIR"; pwd`
 source $GRID_DIR/../gridmix-env
-
-for ((i=0; i < $NUM_OF_LARGE_JOBS_PER_CLASS; i++))
+for ((i=0; i < $NUM_OF_LARGE_JOBS_FOR_ENTROPY_CLASS; i++))
 do
     echo $i
     hod $LARGE_JOB_HOD_OPTIONS --hod.script=$GRID_MIX_HOME/maxent/maxent.large  2>&1
> maxent.large.$i.out &

Modified: hadoop/core/trunk/src/test/gridmix/submissionScripts/sleep_if_too_busy
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/gridmix/submissionScripts/sleep_if_too_busy?rev=630058&r1=630057&r2=630058&view=diff
==============================================================================
--- hadoop/core/trunk/src/test/gridmix/submissionScripts/sleep_if_too_busy (original)
+++ hadoop/core/trunk/src/test/gridmix/submissionScripts/sleep_if_too_busy Thu Feb 21 16:33:56
2008
@@ -1,11 +1,10 @@
 #!/bin/bash
 
 sleep 1
-for ((java_process=$((`ps -ef|grep java|wc|awk '{print $1}'`-1)); \
-      java_process > 60; \
-      java_process=$((`ps -ef|grep java|wc|awk '{print $1}'`-1))))
+for ((java_process=$((`ps -ef|grep java|wc -l`-1)); \
+      java_process > 70; \
+      java_process=$((`ps -ef|grep java|wc -l`-1))))
 do
     sleep 10
     echo $java_process
 done
-



Mime
View raw message