hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ama...@apache.org
Subject svn commit: r1222572 [1/5] - in /hadoop/common/branches/branch-1: ./ ivy/ src/contrib/ src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/ src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/ src/contrib/gridmix/src/test/sys...
Date Fri, 23 Dec 2011 05:38:56 GMT
Author: amarrk
Date: Fri Dec 23 05:38:53 2011
New Revision: 1222572

URL: http://svn.apache.org/viewvc?rev=1222572&view=rev
Log:
MAPREDUCE-2517. Add system tests to Gridmix. (Vinay Thota via amarrk)

Added:
    hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/GridmixSystemTestCase.java
    hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestCPUEmulationForMapsAndReducesWithCustomInterval.java
    hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestCPUEmulationForMapsAndReducesWithDefaultInterval.java
    hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestCPUEmulationForMapsWithCustomInterval.java
    hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestCPUEmulationForMapsWithDefaultInterval.java
    hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestCompressionEmulationEnableForAllTypesOfJobs.java
    hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestCompressionEmulationForCompressInAndUncompressOut.java
    hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestCompressionEmulationForUncompressInAndCompressOut.java
    hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestDisableGridmixEmulationOfHighRam.java
    hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestEmulationOfHDFSAndLocalFSDCFiles.java
    hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestEmulationOfHDFSDCFileUsesMultipleJobs.java
    hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestEmulationOfHDFSDCFilesWithDifferentVisibilities.java
    hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestEmulationOfHighRamAndNormalMRJobs.java
    hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestEmulationOfLocalFSDCFiles.java
    hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixCompressedInputGeneration.java
    hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixCompressionEmulationWithCompressInput.java
    hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixEmulationOfHDFSPrivateDCFile.java
    hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixEmulationOfHDFSPublicDCFile.java
    hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixEmulationOfHighRamJobsCase1.java
    hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixEmulationOfHighRamJobsCase2.java
    hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixEmulationOfHighRamJobsCase3.java
    hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixEmulationOfMultipleHDFSPrivateDCFiles.java
    hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixEmulationOfMultipleHDFSPublicDCFiles.java
    hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith10minTrace.java
    hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith12minTrace.java
    hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith1minTrace.java
    hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith2minStreamingJobTrace.java
    hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith3minStreamingJobTrace.java
    hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith3minTrace.java
    hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith5minStreamingJobTrace.java
    hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith5minTrace.java
    hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith7minTrace.java
    hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestMemEmulForMapsAndReducesWithCustomIntrvl.java
    hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestMemEmulForMapsAndReducesWithDefaultIntrvl.java
    hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestMemEmulForMapsWithCustomHeapMemoryRatio.java
    hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestMemEmulForMapsWithCustomIntrvl.java
    hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestMemEmulForMapsWithDefaultIntrvl.java
    hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/test/
    hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/test/system/
    hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/test/system/GridMixConfig.java
    hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/test/system/GridMixRunMode.java
    hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/test/system/GridmixJobStory.java
    hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/test/system/GridmixJobSubmission.java
    hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/test/system/GridmixJobVerification.java
    hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/test/system/UtilsForGridmix.java
    hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/resources/
    hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/resources/2m_stream_trace.json.gz   (with props)
    hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/resources/3m_stream_trace.json.gz   (with props)
    hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/resources/5m_stream_trace.json.gz   (with props)
    hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/resources/compression_case1_trace.json.gz   (with props)
    hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/resources/compression_case2_trace.json.gz   (with props)
    hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/resources/compression_case3_trace.json.gz   (with props)
    hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/resources/compression_case4_trace.json.gz   (with props)
    hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/resources/cpu_emul_case1.json.gz   (with props)
    hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/resources/cpu_emul_case2.json.gz   (with props)
    hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/resources/distcache_case1_trace.json.gz   (with props)
    hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/resources/distcache_case2_trace.json.gz   (with props)
    hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/resources/distcache_case3_trace.json.gz   (with props)
    hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/resources/distcache_case4_trace.json.gz   (with props)
    hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/resources/distcache_case5_trace.json.gz   (with props)
    hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/resources/distcache_case6_trace.json.gz   (with props)
    hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/resources/distcache_case7_trace.json.gz   (with props)
    hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/resources/distcache_case8_trace.json.gz   (with props)
    hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/resources/distcache_case9_trace.json.gz   (with props)
    hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/resources/highram_mr_jobs_case1.json.gz   (with props)
    hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/resources/highram_mr_jobs_case2.json.gz   (with props)
    hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/resources/highram_mr_jobs_case3.json.gz   (with props)
    hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/resources/highram_mr_jobs_case4.json.gz   (with props)
    hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/resources/mem_emul_case1.json.gz   (with props)
    hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/resources/mem_emul_case2.json.gz   (with props)
    hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/resources/trace_10m.json.gz   (with props)
    hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/resources/trace_12m.json.gz   (with props)
    hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/resources/trace_1m.json.gz   (with props)
    hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/resources/trace_3m.json.gz   (with props)
    hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/resources/trace_5m.json.gz   (with props)
    hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/resources/trace_7m.json.gz   (with props)
Removed:
    hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/GridMixConfig.java
    hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/GridMixRunMode.java
    hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/UtilsForGridmix.java
Modified:
    hadoop/common/branches/branch-1/CHANGES.txt
    hadoop/common/branches/branch-1/build.xml
    hadoop/common/branches/branch-1/ivy/libraries.properties
    hadoop/common/branches/branch-1/src/contrib/build-contrib.xml
    hadoop/common/branches/branch-1/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/DistributedCacheEmulator.java
    hadoop/common/branches/branch-1/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/Gridmix.java
    hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridMixDataGeneration.java
    hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridMixFilePool.java
    hadoop/common/branches/branch-1/src/test/aop/build/aop.xml

Modified: hadoop/common/branches/branch-1/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/CHANGES.txt?rev=1222572&r1=1222571&r2=1222572&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/CHANGES.txt (original)
+++ hadoop/common/branches/branch-1/CHANGES.txt Fri Dec 23 05:38:53 2011
@@ -75,6 +75,8 @@ Release 1.1.0 - unreleased
 
   IMPROVEMENTS
 
+    MAPREDUCE-2517. Add system tests to Gridmix. (Vinay Thota via amarrk)
+
     MAPREDUCE-3008. [Gridmix] Improve cumulative CPU usage emulation for
                     short running tasks. (amarrk)
 

Modified: hadoop/common/branches/branch-1/build.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/build.xml?rev=1222572&r1=1222571&r2=1222572&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/build.xml (original)
+++ hadoop/common/branches/branch-1/build.xml Fri Dec 23 05:38:53 2011
@@ -1594,6 +1594,7 @@
     <mkdir dir="${dist.dir}/share/${name}/contrib"/>
     <mkdir dir="${dist.dir}/share/${name}/webapps"/>
     <mkdir dir="${dist.dir}/share/${name}/templates/conf"/>
+    <mkdir dir="${dist.dir}/native"/>
 
     <copy todir="${dist.dir}/share/${name}/templates/conf" includeEmptyDirs="false">
       <fileset dir="${basedir}/src/packages/templates/conf">

Modified: hadoop/common/branches/branch-1/ivy/libraries.properties
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/ivy/libraries.properties?rev=1222572&r1=1222571&r2=1222572&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/ivy/libraries.properties (original)
+++ hadoop/common/branches/branch-1/ivy/libraries.properties Fri Dec 23 05:38:53 2011
@@ -23,6 +23,7 @@ ant-task.version=2.0.10
 
 asm.version=3.2
 aspectj.version=1.6.5
+aspectj.version=1.6.11
 
 checkstyle.version=4.2
 

Modified: hadoop/common/branches/branch-1/src/contrib/build-contrib.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/contrib/build-contrib.xml?rev=1222572&r1=1222571&r2=1222572&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/contrib/build-contrib.xml (original)
+++ hadoop/common/branches/branch-1/src/contrib/build-contrib.xml Fri Dec 23 05:38:53 2011
@@ -35,6 +35,8 @@
   <property name="src.test" location="${root}/src/test"/>
   <property name="src.test.data" location="${root}/src/test/data"/>
   <!-- Property added for contrib system tests -->
+  <property name="build-fi.dir" location="${hadoop.root}/build-fi"/>
+  <property name="system-test-build-dir" location="${build-fi.dir}/system"/>
   <property name="src.test.system" location="${root}/src/test/system"/>
 
   <property name="src.examples" location="${root}/src/examples"/>
@@ -359,13 +361,84 @@
          <isset property="testcase" />
        </and>
     </condition>
-    <macro-test-runner test.file="${test.all.tests.file}"
-                       classpath="test.system.classpath"
-                       test.dir="${build.test.system}"
-                       fileset.dir="${hadoop.root}/src/contrib/${name}/src/test/system"
-                       hadoop.conf.dir.deployed="${hadoop.conf.dir.deployed}">
-    </macro-test-runner>
-  </target>
+     <property name="test.junit.jvmargs" value="-ea" />
+    <macro-system-test-runner test.file="${test.all.tests.file}"
+                     classpath="test.system.classpath"
+                     test.dir="${build.test.system}"
+                     fileset.dir="${hadoop.root}/src/contrib/${name}/src/test/system"
+                     hadoop.conf.dir.deployed="${hadoop.conf.dir.deployed}">
+  </macro-system-test-runner>
+  </target>
+
+  <macrodef name="macro-system-test-runner">
+    <attribute name="test.file" />
+    <attribute name="classpath" />
+    <attribute name="test.dir" />
+    <attribute name="fileset.dir" />
+    <attribute name="hadoop.conf.dir.deployed" default="" />
+    <sequential>
+      <delete dir="@{test.dir}/data"/>
+      <mkdir dir="@{test.dir}/data"/>
+      <delete dir="@{test.dir}/logs"/>
+      <mkdir dir="@{test.dir}/logs"/>
+      <copy file="${test.src.dir}/hadoop-policy.xml"
+        todir="@{test.dir}/extraconf" />
+      <copy file="${test.src.dir}/fi-site.xml"
+        todir="@{test.dir}/extraconf" />
+      <junit showoutput="${test.output}"
+        printsummary="${test.junit.printsummary}"
+        haltonfailure="${test.junit.haltonfailure}"
+        fork="yes"
+        forkmode="${test.junit.fork.mode}"
+        maxmemory="${test.junit.maxmemory}"
+        dir="${basedir}" timeout="${test.timeout}"
+        errorProperty="tests.failed" failureProperty="tests.failed">
+        <jvmarg value="${test.junit.jvmargs}" />
+        <sysproperty key="java.net.preferIPv4Stack" value="true"/>
+        <sysproperty key="test.build.data" value="@{test.dir}/data"/>
+        <sysproperty key="test.tools.input.dir" value = "${test.tools.input.dir}"/>
+        <sysproperty key="test.cache.data" value="${test.cache.data}"/>
+        <sysproperty key="test.debug.data" value="${test.debug.data}"/>
+        <sysproperty key="hadoop.log.dir" value="@{test.dir}/logs"/>
+        <sysproperty key="test.src.dir" value="@{fileset.dir}"/>
+        <sysproperty key="taskcontroller-path" value="${taskcontroller-path}"/>
+        <sysproperty key="taskcontroller-ugi" value="${taskcontroller-ugi}"/>
+        <sysproperty key="test.build.extraconf" value="@{test.dir}/extraconf" />
+        <sysproperty key="hadoop.policy.file" value="hadoop-policy.xml"/>
+        <sysproperty key="java.library.path"
+          value="${build.native}/lib:${lib.dir}/native/${build.platform}"/>
+        <sysproperty key="install.c++.examples" value="${install.c++.examples}"/>
+        <syspropertyset dynamic="no">
+          <propertyref name="hadoop.tmp.dir"/>
+        </syspropertyset>
+        <!-- set compile.c++ in the child jvm only if it is set -->
+        <syspropertyset dynamic="no">
+          <propertyref name="compile.c++"/>
+        </syspropertyset>
+        <!-- Pass probability specifications to the spawn JVM -->
+        <syspropertyset id="FaultProbabilityProperties">
+          <propertyref regex="fi.*"/>
+        </syspropertyset>
+        <sysproperty key="test.system.hdrc.deployed.hadoopconfdir"
+                     value="@{hadoop.conf.dir.deployed}" />
+        <classpath refid="@{classpath}"/>
+        <formatter type="${test.junit.output.format}" />
+        <batchtest todir="@{test.dir}" unless="testcase">
+          <fileset dir="@{fileset.dir}"
+            excludes="**/${test.exclude}.java aop/** system/**">
+            <patternset>
+              <includesfile name="@{test.file}"/>
+            </patternset>
+          </fileset>
+        </batchtest>
+        <batchtest todir="@{test.dir}" if="testcase">
+          <fileset dir="@{fileset.dir}" includes="**/${testcase}.java"/>
+        </batchtest>
+      </junit>
+      <antcall target="checkfailure"/>
+    </sequential>
+  </macrodef>
+
 
   <target name="checkfailure" if="tests.failed">
     <touch file="${build.contrib.dir}/testsfailed"/>

Modified: hadoop/common/branches/branch-1/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/DistributedCacheEmulator.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/DistributedCacheEmulator.java?rev=1222572&r1=1222571&r2=1222572&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/DistributedCacheEmulator.java (original)
+++ hadoop/common/branches/branch-1/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/DistributedCacheEmulator.java Fri Dec 23 05:38:53 2011
@@ -360,7 +360,7 @@ class DistributedCacheEmulator {
    * @return true if the path provided is of a local file system based
    *              distributed cache file
    */
-  private boolean isLocalDistCacheFile(String filePath, String user,
+  protected static boolean isLocalDistCacheFile(String filePath, String user,
                                        boolean visibility) {
     return (!visibility && filePath.contains(user + "/.staging"));
   }

Modified: hadoop/common/branches/branch-1/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/Gridmix.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/Gridmix.java?rev=1222572&r1=1222571&r2=1222572&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/Gridmix.java (original)
+++ hadoop/common/branches/branch-1/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/Gridmix.java Fri Dec 23 05:38:53 2011
@@ -124,7 +124,7 @@ public class Gridmix extends Configured 
     summarizer = new Summarizer(args);
   }
   
-  Gridmix() {
+  public Gridmix() {
     summarizer = new Summarizer();
   }
   

Added: hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/GridmixSystemTestCase.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/GridmixSystemTestCase.java?rev=1222572&view=auto
==============================================================================
--- hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/GridmixSystemTestCase.java (added)
+++ hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/GridmixSystemTestCase.java Fri Dec 23 05:38:53 2011
@@ -0,0 +1,234 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred.gridmix;
+
+import org.apache.commons.logging.LogFactory;
+import org.apache.commons.logging.Log;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapreduce.test.system.MRCluster;
+import org.apache.hadoop.mapreduce.test.system.JTProtocol;
+import org.apache.hadoop.mapreduce.test.system.JTClient;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.gridmix.test.system.GridmixJobSubmission;
+import org.apache.hadoop.mapred.gridmix.test.system.GridmixJobVerification;
+import org.apache.hadoop.mapred.gridmix.test.system.GridMixRunMode;
+import org.apache.hadoop.mapred.gridmix.test.system.GridMixConfig;
+import org.apache.hadoop.mapred.gridmix.test.system.UtilsForGridmix;
+import org.apache.hadoop.mapred.gridmix.test.system.GridmixJobStory;
+import org.apache.hadoop.tools.rumen.ZombieJob;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.mapreduce.JobID;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+
+import java.util.Iterator;
+import java.util.Map;
+import java.util.List;
+import java.util.Set;
+import java.io.IOException;
+import org.junit.Assert;
+
+/**
+ * Run and verify the Gridmix jobs for given a trace.
+ */
+public class GridmixSystemTestCase {
+  private static final Log LOG = 
+      LogFactory.getLog(GridmixSystemTestCase.class);
+  protected static Configuration  conf = new Configuration();
+  protected static MRCluster cluster;
+  protected static int cSize;
+  protected static JTClient jtClient;
+  protected static JTProtocol rtClient;
+  protected static Path gridmixDir;
+  protected static Map<String, String> map;
+  protected static GridmixJobSubmission gridmixJS;
+  protected static GridmixJobVerification gridmixJV;
+  protected static List<JobID> jobids;
+  
+  @BeforeClass
+  public static void before() throws Exception {
+    String [] excludeExpList = {"java.net.ConnectException", 
+                                "java.io.IOException",
+                               };
+    cluster = MRCluster.createCluster(conf);
+    cluster.setExcludeExpList(excludeExpList);
+    cluster.setUp();
+    cSize = cluster.getTTClients().size();
+    jtClient = cluster.getJTClient();
+    rtClient = jtClient.getProxy();
+    gridmixDir = new Path("herriot-gridmix");
+    UtilsForGridmix.createDirs(gridmixDir, rtClient.getDaemonConf());
+    map = UtilsForGridmix.getMRTraces(rtClient.getDaemonConf());
+  }
+
+  @AfterClass
+  public static void after() throws Exception {
+    UtilsForGridmix.cleanup(gridmixDir, rtClient.getDaemonConf());
+    org.apache.hadoop.fs.FileUtil.fullyDelete(new java.io.File(System.
+        getProperty("java.io.tmpdir") + "/gridmix-st/"));
+    cluster.tearDown();
+
+    /* Clean up the proxy user directories if gridmix run with 
+      RoundRobinUserResovler mode.*/
+    // The below code commented due to a bug in mapreduce framework.
+    /*if (gridmixJV != null 
+       && gridmixJV.getJobUserResolver().contains("RoundRobin")) {
+       List<String> proxyUsers = 
+           UtilsForGridmix.listProxyUsers(gridmixJS.getJobConf(),
+           UserGroupInformation.getLoginUser().getShortUserName());
+       for (int index = 0; index < proxyUsers.size(); index++){
+         UtilsForGridmix.cleanup(new Path("hdfs:///user/" + 
+             proxyUsers.get(index)), 
+             rtClient.getDaemonConf());
+       }
+    }*/
+  }
+  
+  /**
+   * Run the gridmix with specified runtime parameters and 
+   * verify the jobs the after completion of execution.
+   * @param runtimeValues - common runtime arguments for gridmix.
+   * @param otherValues - test specific runtime arguments for gridmix.
+   * @param tracePath - path of a trace file.
+   * @throws Exception - if an exception occurs.
+   */
+  public static void runGridmixAndVerify(String[] runtimeValues, 
+     String [] otherValues, String tracePath) throws Exception {
+     runGridmixAndVerify(runtimeValues, otherValues, tracePath , 
+         GridMixRunMode.DATA_GENERATION_AND_RUN_GRIDMIX.getValue());
+  }
+  
+  /**
+   * Run the gridmix with specified runtime parameters and
+   * verify the jobs the after completion of execution.
+   * @param runtimeValues - common runtime arguments for gridmix.
+   * @param otherValues - test specific runtime arguments for gridmix.
+   * @param tracePath - path of a trace file.
+   * @param mode - 1 for data generation, 2 for run the gridmix and 3 for
+   * data generation and run the gridmix.
+   * @throws Exception - if an exception occurs.
+   */
+  public static void runGridmixAndVerify(String [] runtimeValues, 
+      String [] otherValues, String tracePath, int mode) throws Exception {
+    List<JobID> jobids = runGridmix(runtimeValues, otherValues, mode);
+    gridmixJV = new GridmixJobVerification(new Path(tracePath), 
+                                           gridmixJS.getJobConf(), jtClient);
+    gridmixJV.verifyGridmixJobsWithJobStories(jobids);  
+  }
+
+  /**
+   * Run the gridmix with user specified mode.
+   * @param runtimeValues - common runtime parameters for gridmix.
+   * @param otherValues - test specifix runtime parameters for gridmix.
+   * @param mode -  1 for data generation, 2 for run the gridmix and 3 for
+   * data generation and run the gridmix.
+   * @return - list of gridmix job ids.
+   * @throws Exception - if an exception occurs.
+   */
+  public static List<JobID> runGridmix(String[] runtimeValues, 
+     String[] otherValues, int mode) throws Exception {
+    gridmixJS = new GridmixJobSubmission(rtClient.getDaemonConf(),
+       jtClient, gridmixDir);
+    gridmixJS.submitJobs(runtimeValues, otherValues, mode);
+    LOG.info("Gridmix JobCount:" + gridmixJS.getGridmixJobCount());
+    List<JobID> jobids = 
+        UtilsForGridmix.listGridmixJobIDs(jtClient.getClient(), 
+                                          gridmixJS.getGridmixJobCount());
+    return jobids;
+  }
+  
+  /**
+   * get the trace file based on given regular expression.
+   * @param regExp - trace file file pattern. 
+   * @return - trace file as string.
+   * @throws IOException - if an I/O error occurs.
+   */
+  public static String getTraceFile(String regExp) throws IOException {
+    List<String> listTraces = UtilsForGridmix.listMRTraces(
+        rtClient.getDaemonConf());
+    Iterator<String> ite = listTraces.iterator();
+    while(ite.hasNext()) {
+      String traceFile = ite.next();
+      if (traceFile.indexOf(regExp)>=0) {
+        return traceFile;
+      }
+    }
+    return null;
+  }
+
+  /**
+   * Validate the task memory parameters.
+   * @param tracePath - trace file.
+   * @param isTraceHasHighRamJobs - true if trace has high ram job(s) 
+   *                                otherwise its false 
+   */
+  @SuppressWarnings("deprecation")
+  public static void validateTaskMemoryParamters(String tracePath,
+      boolean isTraceHasHighRamJobs) throws IOException {
+    if (isTraceHasHighRamJobs) {
+      GridmixJobStory gjs = new GridmixJobStory(new Path(tracePath),
+                                                rtClient.getDaemonConf());
+      Set<JobID> jobids = gjs.getZombieJobs().keySet();
+      boolean isHighRamFlag = false;
+      for (JobID jobid :jobids) {
+        ZombieJob zombieJob = gjs.getZombieJobs().get(jobid);
+        JobConf origJobConf = zombieJob.getJobConf();
+        int origMapFactor =
+            GridmixJobVerification.getMapFactor(origJobConf);
+        int origReduceFactor =
+            GridmixJobVerification.getReduceFactor(origJobConf);
+        if (origMapFactor >= 2 || origReduceFactor >= 2) {
+          isHighRamFlag = true;
+          long TaskMapMemInMB =
+              GridmixJobVerification.getScaledTaskMemInMB(
+                      GridMixConfig.JOB_MAP_MEMORY_MB,
+                      GridMixConfig.CLUSTER_MAP_MEMORY,
+                      origJobConf, rtClient.getDaemonConf());
+
+          long TaskReduceMemInMB =
+              GridmixJobVerification.getScaledTaskMemInMB(
+                      GridMixConfig.JOB_REDUCE_MEMORY_MB,
+                      GridMixConfig.CLUSTER_REDUCE_MEMORY,
+                      origJobConf, rtClient.getDaemonConf());
+          long taskMapLimitInMB =
+              conf.getLong(GridMixConfig.CLUSTER_MAX_MAP_MEMORY,
+                           JobConf.DISABLED_MEMORY_LIMIT);
+
+          long taskReduceLimitInMB =
+              conf.getLong(GridMixConfig.CLUSTER_MAX_REDUCE_MEMORY,
+                           JobConf.DISABLED_MEMORY_LIMIT);
+
+          GridmixJobVerification.verifyMemoryLimits(TaskMapMemInMB,
+                                                    taskMapLimitInMB);
+          GridmixJobVerification.verifyMemoryLimits(TaskReduceMemInMB,
+                                                    taskReduceLimitInMB);
+        }
+      }
+      Assert.assertTrue("Trace doesn't have atleast one high ram job.",
+                        isHighRamFlag);
+    }
+  }
+
+  public static boolean isLocalDistCache(String fileName, String userName, 
+                                         boolean visibility) {
+    return DistributedCacheEmulator.isLocalDistCacheFile(fileName, 
+                                                         userName, visibility);
+  }
+}
+

Added: hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestCPUEmulationForMapsAndReducesWithCustomInterval.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestCPUEmulationForMapsAndReducesWithCustomInterval.java?rev=1222572&view=auto
==============================================================================
--- hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestCPUEmulationForMapsAndReducesWithCustomInterval.java (added)
+++ hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestCPUEmulationForMapsAndReducesWithCustomInterval.java Fri Dec 23 05:38:53 2011
@@ -0,0 +1,107 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred.gridmix;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.mapred.gridmix.test.system.GridMixConfig;
+import org.apache.hadoop.mapred.gridmix.test.system.GridMixRunMode;
+import org.apache.hadoop.mapred.gridmix.test.system.UtilsForGridmix;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.junit.Test;
+import org.junit.Assert;
+
+/**
+ * Test cpu emulation with default interval for gridmix jobs 
+ * against different input data, submission policies and user resolvers.
+ * Verify the cpu resource metrics of both maps and reduces phase of
+ * Gridmix jobs with their corresponding original job in the input trace.
+ */
+public class TestCPUEmulationForMapsAndReducesWithCustomInterval 
+    extends GridmixSystemTestCase { 
+  private static final Log LOG = LogFactory
+      .getLog("TestCPUEmulationForMapsAndReducesWithCustomInterval.class");
+  int execMode = GridMixRunMode.DATA_GENERATION_AND_RUN_GRIDMIX.getValue();
+
+ /**
+   * Generate compressed input and run {@link Gridmix} by turning on the 
+   * cpu emulation feature with default setting. The {@link Gridmix} 
+   * should use the following runtime parameters.
+   * Submission Policy : STRESS, UserResovler: RoundRobinUserResolver. 
+   * Once the {@link Gridmix} run is complete, verify cpu resource metrics of 
+   * {@link Gridmix} jobs with their corresponding original job in a trace.
+   * @throws Exception - if an error occurs.
+   */
+  @Test
+  public void  testCPUEmulationForMapsAndReducesWithCompressedInputCase7() 
+      throws Exception {
+    final long inputSizeInMB = 1024 * 7;
+    String tracePath = getTraceFile("cpu_emul_case2");
+    Assert.assertNotNull("Trace file not found!", tracePath);
+    String [] runtimeValues = 
+            { "LOADJOB",
+              RoundRobinUserResolver.class.getName(),
+              "STRESS",
+              inputSizeInMB + "m",
+              "file://" + UtilsForGridmix.getProxyUsersFile(conf),
+              tracePath};
+
+    String [] otherArgs = { 
+            "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false",
+            "-D", JobContext.JOB_CANCEL_DELEGATION_TOKEN + "=false",
+            "-D", GridMixConfig.GRIDMIX_CPU_CUSTOM_INTERVAL + "=0.35F",
+            "-D", GridMixConfig.GRIDMIX_CPU_EMULATION + "=" + 
+                  GridMixConfig.GRIDMIX_CPU_EMULATION_PLUGIN};
+
+    runGridmixAndVerify(runtimeValues, otherArgs, tracePath, execMode); 
+  }
+  
+  /**
+   * Generate uncompressed input and run {@link Gridmix} by turning on the 
+   * cpu emulation feature with default setting. The {@link Gridmix} 
+   * should use the following runtime parameters.
+   * Submission Policy : SERIAL, UserResovler: SubmitterUserResolver 
+   * Once the {@link Gridmix} run is complete, verify cpu resource metrics of 
+   * {@link Gridmix} jobs with their corresponding original job in a trace.
+   * @throws Exception - if an error occurs.
+   */
+  @Test
+  public void  testCPUEmulatonForMapsAndReducesWithUncompressedInputCase8() 
+      throws Exception {
+    final long inputSizeInMB = cSize * 300;
+    String tracePath = getTraceFile("cpu_emul_case2");
+    Assert.assertNotNull("Trace file not found.", tracePath);
+    String [] runtimeValues = 
+              { "LOADJOB", 
+                SubmitterUserResolver.class.getName(), 
+                "SERIAL", 
+                inputSizeInMB + "m",
+                tracePath};
+
+    String [] otherArgs = {
+            "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false",
+            "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false",
+            "-D", JobContext.JOB_CANCEL_DELEGATION_TOKEN + "=false",
+            "-D", GridMixConfig.GRIDMIX_CPU_CUSTOM_INTERVAL + "=0.4F",
+            "-D", GridMixConfig.GRIDMIX_CPU_EMULATION + "=" + 
+                  GridMixConfig.GRIDMIX_CPU_EMULATION_PLUGIN     };
+
+    runGridmixAndVerify(runtimeValues, otherArgs, tracePath, execMode); 
+  }
+}
+

Added: hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestCPUEmulationForMapsAndReducesWithDefaultInterval.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestCPUEmulationForMapsAndReducesWithDefaultInterval.java?rev=1222572&view=auto
==============================================================================
--- hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestCPUEmulationForMapsAndReducesWithDefaultInterval.java (added)
+++ hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestCPUEmulationForMapsAndReducesWithDefaultInterval.java Fri Dec 23 05:38:53 2011
@@ -0,0 +1,106 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred.gridmix;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.mapred.gridmix.test.system.GridMixConfig;
+import org.apache.hadoop.mapred.gridmix.test.system.GridMixRunMode;
+import org.apache.hadoop.mapred.gridmix.test.system.UtilsForGridmix;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.junit.Test;
+import org.junit.Assert;
+
+/**
+ * Test cpu emulation with default interval for gridmix jobs 
+ * against different input data, submission policies and user resolvers.
+ * Verify the cpu resource metrics for both maps and reduces of
+ * Gridmix jobs with their corresponding original job in the input trace.
+ */
+public class TestCPUEmulationForMapsAndReducesWithDefaultInterval 
+                                            extends GridmixSystemTestCase { 
+  private static final Log LOG = 
+          LogFactory.getLog(
+              "TestCPUEmulationForMapsAndReducesWithDefaultInterval.class");
+  int execMode = GridMixRunMode.DATA_GENERATION_AND_RUN_GRIDMIX.getValue();
+
+ /**
+   * Generate compressed input and run {@link Gridmix} by turning on the 
+   * cpu emulation feature with default setting. The {@link Gridmix} 
+   * should use the following runtime parameters.
+   * Submission Policy : REPLAY, UserResovler: RoundRobinUserResolver. 
+   * Once the {@link Gridmix} run is complete, verify cpu resource metrics of 
+   * {@link Gridmix} jobs with their corresponding original jobs in the trace.
+   * @throws Exception - if an error occurs.
+   */
+  @Test
+  public void  testCPUEmulationForMapsAndReducesWithCompressedInputCase5() 
+      throws Exception {
+    final long inputSizeInMB = 7168;
+    String tracePath = getTraceFile("cpu_emul_case2");
+    Assert.assertNotNull("Trace file not found!", tracePath);
+    String [] runtimeValues = 
+            { "LOADJOB",
+              RoundRobinUserResolver.class.getName(),
+              "REPLAY",
+              inputSizeInMB + "m",
+              "file://" + UtilsForGridmix.getProxyUsersFile(conf),
+              tracePath};
+
+    String [] otherArgs = { 
+            "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false",
+            "-D", JobContext.JOB_CANCEL_DELEGATION_TOKEN + "=false",
+            "-D", GridMixConfig.GRIDMIX_CPU_EMULATION + "=" + 
+                  GridMixConfig.GRIDMIX_CPU_EMULATION_PLUGIN};
+
+    runGridmixAndVerify(runtimeValues, otherArgs, tracePath, execMode); 
+  }
+  
+  /**
+   * Generate uncompressed input and run {@link Gridmix} by turning on the 
+   * cpu emulation feature with default settings. The {@link Gridmix} 
+   * should use the following runtime parameters.
+   * Submission Policy : STRESS, UserResovler: SubmitterUserResolver 
+   * Once the Gridmix run is complete, verify cpu resource metrics of 
+   * {@link Gridmix} jobs with their corresponding original jobs in the trace.
+   * @throws Exception - if an error occurs.
+   */
+  @Test
+  public void  testCPUEmulatonForMapsAndReducesWithUncompressedInputCase6() 
+      throws Exception {
+    final long inputSizeInMB = cSize * 400;
+    String tracePath = getTraceFile("cpu_emul_case2");
+    Assert.assertNotNull("Trace file not found!", tracePath);
+    String [] runtimeValues = 
+              { "LOADJOB", 
+                SubmitterUserResolver.class.getName(), 
+                "STRESS",
+                inputSizeInMB + "m",
+                tracePath};
+
+    String [] otherArgs = {
+            "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false",
+            "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false",
+            "-D", JobContext.JOB_CANCEL_DELEGATION_TOKEN + "=false",
+            "-D", GridMixConfig.GRIDMIX_CPU_EMULATION + "=" + 
+                  GridMixConfig.GRIDMIX_CPU_EMULATION_PLUGIN     };
+    
+    runGridmixAndVerify(runtimeValues, otherArgs, tracePath, execMode); 
+  }
+}
+

Added: hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestCPUEmulationForMapsWithCustomInterval.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestCPUEmulationForMapsWithCustomInterval.java?rev=1222572&view=auto
==============================================================================
--- hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestCPUEmulationForMapsWithCustomInterval.java (added)
+++ hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestCPUEmulationForMapsWithCustomInterval.java Fri Dec 23 05:38:53 2011
@@ -0,0 +1,104 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred.gridmix;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.mapred.gridmix.test.system.GridMixConfig;
+import org.apache.hadoop.mapred.gridmix.test.system.GridMixRunMode;
+import org.apache.hadoop.mapred.gridmix.test.system.UtilsForGridmix;
+import org.junit.Test;
+import org.junit.Assert;
+
+/**
+ * Test the {@link Gridmix} cpu emulation with custom interval for 
+ * gridmix jobs against different input data, submission policies and 
+ * user resolvers. Verify the map phase cpu metrics of gridmix jobs 
+ * against their original job in the trace. 
+ */
+public class TestCPUEmulationForMapsWithCustomInterval 
+                                            extends GridmixSystemTestCase {
+  private static final Log LOG = 
+      LogFactory.getLog("TestCPUEmulationForMapsWithCustomInterval.class");
+  int execMode = GridMixRunMode.DATA_GENERATION_AND_RUN_GRIDMIX.getValue();
+
+  /**
+   * Generate compressed input and run {@link Gridmix} by turning on 
+   * cpu emulation feature with custom setting. The {@link Gridmix} should 
+   * use the following runtime parameters while running gridmix jobs.
+   * Submission Policy : STRESS, User Resolver Mode : SumitterUserResolver
+   * Once {@link Gridmix} run is complete, verify maps phase cpu resource 
+   * metrics of {@link Gridmix} jobs with their corresponding original
+   * in the trace.
+   * @throws Exception - if an error occurs.
+   */
+  @Test
+  public void  testCPUEmulatonForMapsWithCompressedInputCase3() 
+      throws Exception { 
+    final long inputSizeInMB = 1024 * 7;
+    String tracePath = getTraceFile("cpu_emul_case1");
+    Assert.assertNotNull("Trace file not found!", tracePath);
+    String [] runtimeValues = {"LOADJOB",
+                               SubmitterUserResolver.class.getName(),
+                               "STRESS",
+                               inputSizeInMB + "m",
+                               tracePath};
+
+    String [] otherArgs = {
+        "-D", GridMixConfig.GRIDMIX_CPU_EMULATION + "=" +
+              GridMixConfig.GRIDMIX_CPU_EMULATION_PLUGIN,
+        "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false",
+        "-D", GridMixConfig.GRIDMIX_CPU_CUSTOM_INTERVAL + "=0.25F"};
+
+    runGridmixAndVerify(runtimeValues, otherArgs, tracePath, execMode);
+  }
+
+  /**
+   * Generate uncompressed input and run {@link Gridmix} by turning on 
+   * cpu emulation feature with custom settings. The {@link Gridmix} 
+   * should use the following runtime paramters while running gridmix jobs.
+   * Submission Policy: REPLAY  User Resolver Mode: RoundRobinUserResolver
+   * Once {@link Gridmix} run is complete, verify the map phase cpu resource 
+   * metrics of {@link Gridmix} jobs with their corresponding jobs
+   * in the original trace.
+   * @throws Exception - if an error occurs.
+   */
+  @Test
+  public void testCPUEmulatonForMapsUnCompressedInputCase4() 
+      throws Exception { 
+    final long inputSizeInMB = cSize * 200;
+    String tracePath = getTraceFile("cpu_emul_case1");
+    Assert.assertNotNull("Trace file not found!", tracePath);
+    String [] runtimeValues = 
+           {"LOADJOB",
+            RoundRobinUserResolver.class.getName(),
+            "REPLAY",
+            inputSizeInMB + "m",
+            "file://" + UtilsForGridmix.getProxyUsersFile(conf),
+            tracePath};
+
+    String [] otherArgs = {
+        "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false",
+        "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false",
+        "-D", GridMixConfig.GRIDMIX_CPU_EMULATION + "=" + 
+              GridMixConfig.GRIDMIX_CPU_EMULATION_PLUGIN,
+        "-D", GridMixConfig.GRIDMIX_CPU_CUSTOM_INTERVAL + "=0.35F"};
+
+    runGridmixAndVerify(runtimeValues, otherArgs, tracePath, execMode);
+  }
+}

Added: hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestCPUEmulationForMapsWithDefaultInterval.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestCPUEmulationForMapsWithDefaultInterval.java?rev=1222572&view=auto
==============================================================================
--- hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestCPUEmulationForMapsWithDefaultInterval.java (added)
+++ hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestCPUEmulationForMapsWithDefaultInterval.java Fri Dec 23 05:38:53 2011
@@ -0,0 +1,103 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred.gridmix;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.mapred.gridmix.test.system.GridMixConfig;
+import org.apache.hadoop.mapred.gridmix.test.system.GridMixRunMode;
+import org.apache.hadoop.mapred.gridmix.test.system.UtilsForGridmix;
+import org.junit.Test;
+import org.junit.Assert;
+
+/**
+ * Test the {@link Gridmix} cpu emulation with default settings for 
+ * gridmix jobs against different input data, submission policies and 
+ * user resolvers. Verify the map phase cpu metrics of gridmix jobs 
+ * against their original jobs in the trace. 
+ */
+public class TestCPUEmulationForMapsWithDefaultInterval 
+    extends GridmixSystemTestCase { 
+  private static final Log LOG = 
+      LogFactory.getLog("TestCPUEmulationForMapsWithDefaultInterval.class");
+  int execMode = GridMixRunMode.DATA_GENERATION_AND_RUN_GRIDMIX.getValue();
+
+  /**
+   * Generate compressed input and run {@link Gridmix} by turning on cpu 
+   * emulation feature with default settings. The {@link Gridmix} should 
+   * use the following runtime parameters while running the gridmix jobs.
+   * Submission Policy: STRESS, UserResolver: SubmitterUserResolver. 
+   * Once the {@link Gridmix} run is complete, verify map phase cpu metrics of 
+   * {@link Gridmix} jobs with their corresponding original job in a trace.
+   * @throws Exception - if an error occurs.
+   */
+  @Test
+  public void testCPUEmulatonForMapsWithCompressedInputCase1() 
+      throws Exception {
+    final long inputSizeInMB = 1024 * 6;
+    String tracePath = getTraceFile("cpu_emul_case1");
+    Assert.assertNotNull("Trace file not found!", tracePath);
+    String [] runtimeValues = { "LOADJOB", 
+                                SubmitterUserResolver.class.getName(), 
+                                "STRESS", 
+                                inputSizeInMB + "m", 
+                                tracePath};
+
+    String [] otherArgs = { 
+            "-D", GridMixConfig.GRIDMIX_CPU_EMULATION + "=" + 
+                  GridMixConfig.GRIDMIX_CPU_EMULATION_PLUGIN,
+            "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false",
+            "-D", GridMixConfig.GRIDMIX_HIGH_RAM_JOB_ENABLE + "=false"};
+
+    runGridmixAndVerify(runtimeValues, otherArgs, tracePath, execMode); 
+  }
+  
+  /**
+   * Generate uncompressed input and run {@link Gridmix} by turning on 
+   * cpu emulation feature with default settings. The {@link Gridmix} 
+   * should use the following runtime parameters while running Gridmix jobs.
+   * Submission Policy: REPLAY, UserResolver: RoundRobinUserResolver
+   * Once the Gridmix run is complete, verify cpu resource metrics of 
+   * {@link Gridmix} jobs with their corresponding original job in a trace.
+   * @throws Exception - if an error occurs.
+   */
+  @Test
+  public void testCPUEmulatonForMapsWithUnCompressedInputCase2() 
+      throws Exception { 
+    final long inputSizeInMB = cSize * 200;
+    String tracePath = getTraceFile("cpu_emul_case1");
+    Assert.assertNotNull("Trace file not found!", tracePath);
+    String [] runtimeValues = 
+            { "LOADJOB",
+              RoundRobinUserResolver.class.getName(),
+              "REPLAY",
+              inputSizeInMB + "m",
+              "file://" + UtilsForGridmix.getProxyUsersFile(conf),
+              tracePath};
+
+    String [] otherArgs = { 
+            "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false",
+            "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false",
+            "-D", GridMixConfig.GRIDMIX_HIGH_RAM_JOB_ENABLE + "=false",
+            "-D", GridMixConfig.GRIDMIX_CPU_EMULATION + "=" + 
+                  GridMixConfig.GRIDMIX_CPU_EMULATION_PLUGIN};
+
+    runGridmixAndVerify(runtimeValues, otherArgs, tracePath, 
+           GridMixRunMode.DATA_GENERATION_AND_RUN_GRIDMIX.getValue());
+  }
+}

Added: hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestCompressionEmulationEnableForAllTypesOfJobs.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestCompressionEmulationEnableForAllTypesOfJobs.java?rev=1222572&view=auto
==============================================================================
--- hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestCompressionEmulationEnableForAllTypesOfJobs.java (added)
+++ hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestCompressionEmulationEnableForAllTypesOfJobs.java Fri Dec 23 05:38:53 2011
@@ -0,0 +1,96 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred.gridmix;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.mapred.gridmix.test.system.GridMixConfig;
+import org.apache.hadoop.mapred.gridmix.test.system.GridMixRunMode;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Verify the compression emulation for all the jobs in the trace 
+ * irrespective of compressed inputs.
+ */
+public class TestCompressionEmulationEnableForAllTypesOfJobs 
+    extends GridmixSystemTestCase { 
+  private static final Log LOG = 
+      LogFactory.getLog(
+          "TestCompressionEmulationEnableForAllTypesOfJobs.class");
+
+  /**
+   *  Generate compressed input data and verify the compression emulation
+   *  for all the jobs in the trace irrespective of whether the original
+   *  job uses the compressed input or not.Also use the custom compression
+   *  ratios for map input, map output and reduce output.
+   * @throws Exception - if an error occurs.
+   */
+  @Test
+  public void testInputCompressionEmualtionEnableForAllJobsWithCustomRatios() 
+      throws Exception { 
+    final long inputSizeInMB = 1024 * 6;
+    final String tracePath = getTraceFile("compression_case4_trace");
+    Assert.assertNotNull("Trace file has not found.", tracePath);
+    final String [] runtimeValues = {"LOADJOB",
+                                     SubmitterUserResolver.class.getName(),
+                                     "REPLAY",
+                                     inputSizeInMB + "m",
+                                     tracePath};
+
+    final String [] otherArgs = {
+        "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false",
+        "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=true",
+        "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false",
+        "-D", GridMixConfig.GRIDMIX_INPUT_DECOMPRESS_ENABLE + "=true",
+        "-D", GridMixConfig.GRIDMIX_INPUT_COMPRESS_RATIO + "=0.46",
+        "-D", GridMixConfig.GRIDMIX_INTERMEDIATE_COMPRESSION_RATIO + "=0.35",
+        "-D", GridMixConfig.GRIDMIX_OUTPUT_COMPRESSION_RATIO + "=0.36"
+    };
+
+    runGridmixAndVerify(runtimeValues, otherArgs, tracePath, 
+        GridMixRunMode.DATA_GENERATION_AND_RUN_GRIDMIX.getValue());
+  }
+  
+  /**
+   *  Use existing compressed input data and turn off the compression 
+   *  emulation. Verify the compression emulation whether it uses 
+   *  by the jobs or not.
+   * @throws Exception - if an error occurs.
+   */
+  @Test
+  public void testInputCompressionEmulationDisableAllJobs() 
+      throws Exception { 
+     final String tracePath = getTraceFile("compression_case4_trace");
+     Assert.assertNotNull("Trace file has not found.", tracePath);
+     final String [] runtimeValues = {"LOADJOB",
+                                      SubmitterUserResolver.class.getName(),
+                                      "SERIAL",
+                                      tracePath};
+
+    final String [] otherArgs = {
+        "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false",
+        "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false",
+        "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false"
+    };
+
+    runGridmixAndVerify(runtimeValues, otherArgs, tracePath, 
+                        GridMixRunMode.RUN_GRIDMIX.getValue());
+  }  
+}
+

Added: hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestCompressionEmulationForCompressInAndUncompressOut.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestCompressionEmulationForCompressInAndUncompressOut.java?rev=1222572&view=auto
==============================================================================
--- hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestCompressionEmulationForCompressInAndUncompressOut.java (added)
+++ hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestCompressionEmulationForCompressInAndUncompressOut.java Fri Dec 23 05:38:53 2011
@@ -0,0 +1,98 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred.gridmix;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.mapred.gridmix.test.system.GridMixConfig;
+import org.apache.hadoop.mapred.gridmix.test.system.GridMixRunMode;
+import org.apache.hadoop.mapred.gridmix.test.system.UtilsForGridmix;
+import org.junit.Assert;
+import org.junit.Test;
+/**
+ * Verify the gridmix jobs compression ratio's of input, 
+ * intermediate input and with default/custom ratios.Also verify
+ * the compressed output file format is enabled or not.
+ *
+ */
+public class TestCompressionEmulationForCompressInAndUncompressOut 
+    extends GridmixSystemTestCase { 
+  private static final Log LOG = 
+      LogFactory.getLog(
+          "TestCompressionEmulationForCompressInAndUncompressOut.class");
+  final long inputSizeInMB = 1024 * 6;
+
+  /**
+   * Generate a compressed input data and verify the compression ratios 
+   * of map input and map output against default compression ratios 
+   * and also verify the whether the compressed output file output format 
+   * is enabled or not.
+   * @throws Exception -if an error occurs.
+   */
+  @Test
+  public void testCompressionEmulationOfCompressedInputWithDefaultRatios() 
+      throws Exception {
+    final String tracePath = getTraceFile("compression_case2_trace");
+    Assert.assertNotNull("Trace file has not found.", tracePath);
+    final String [] runtimeValues = {"LOADJOB",
+                                     SubmitterUserResolver.class.getName(),
+                                     "STRESS",
+                                     inputSizeInMB + "m",
+                                     tracePath};
+
+    final String [] otherArgs = {
+        "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false",
+        "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false",
+        "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=true"
+    };
+
+    runGridmixAndVerify(runtimeValues, otherArgs, tracePath, 
+        GridMixRunMode.DATA_GENERATION_AND_RUN_GRIDMIX.getValue());
+  }
+  
+  /**
+   * Use existing compressed input data and verify the compression ratios 
+   * of input and intermediate input against custom compression ratios 
+   * and also verify the compressed output file output format is enabled or not.
+   * @throws Exception -if an error occurs.
+   */
+  @Test
+  public void testCompressionEmulationOfCompressedInputWithCustomRatios() 
+      throws Exception {
+    final String tracePath = getTraceFile("compression_case2_trace");
+    Assert.assertNotNull("Trace file has not found.", tracePath);
+    UtilsForGridmix.cleanup(gridmixDir, rtClient.getDaemonConf());
+    final String [] runtimeValues = {"LOADJOB",
+                                     SubmitterUserResolver.class.getName(),
+                                     "STRESS",
+                                     inputSizeInMB + "m",
+                                     tracePath};
+
+    final String [] otherArgs = {
+        "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false",
+        "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=true",
+        "-D", GridMixConfig.GRIDMIX_INPUT_DECOMPRESS_ENABLE + "=true",
+        "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false",
+        "-D", GridMixConfig.GRIDMIX_INPUT_COMPRESS_RATIO + "=0.58",
+        "-D", GridMixConfig.GRIDMIX_INTERMEDIATE_COMPRESSION_RATIO + "=0.42"
+    };
+
+    runGridmixAndVerify(runtimeValues, otherArgs, tracePath, 
+        GridMixRunMode.DATA_GENERATION_AND_RUN_GRIDMIX.getValue());
+  }
+}
+

Added: hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestCompressionEmulationForUncompressInAndCompressOut.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestCompressionEmulationForUncompressInAndCompressOut.java?rev=1222572&view=auto
==============================================================================
--- hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestCompressionEmulationForUncompressInAndCompressOut.java (added)
+++ hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestCompressionEmulationForUncompressInAndCompressOut.java Fri Dec 23 05:38:53 2011
@@ -0,0 +1,93 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred.gridmix;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.mapred.gridmix.test.system.GridMixConfig;
+import org.apache.hadoop.mapred.gridmix.test.system.GridMixRunMode;
+import org.apache.hadoop.mapred.gridmix.test.system.UtilsForGridmix;
+import org.junit.Assert;
+import org.junit.Test;
+/**
+ * Verify the gridmix jobs compression ratio's of reduce output and 
+ * with default and custom ratios.
+ */
+public class TestCompressionEmulationForUncompressInAndCompressOut
+   extends GridmixSystemTestCase { 
+   private static final Log LOG = 
+       LogFactory.getLog(
+           "TestCompressionEmulationForUncompressInAndCompressOut.class");
+   final long inputSizeInMB = 1024 * 6;
+
+  /**
+   * Generate a uncompressed input data and verify the compression ratios 
+   * of reduce output against default output compression ratio.
+   * @throws Exception -if an error occurs.
+   */
+  @Test
+  public void testCompressionEmulationOfCompressedOuputWithDefaultRatios() 
+      throws Exception { 
+    final String tracePath = getTraceFile("compression_case3_trace");
+    Assert.assertNotNull("Trace file has not found.", tracePath);
+    final String [] runtimeValues = 
+                     {"LOADJOB",
+                      RoundRobinUserResolver.class.getName(),
+                      "REPLAY",
+                      inputSizeInMB + "m",
+                      "file://" + UtilsForGridmix.getProxyUsersFile(conf),
+                      tracePath};
+
+    final String [] otherArgs = {
+        "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false",
+        "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false",
+        "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=true"
+    };
+
+    runGridmixAndVerify(runtimeValues, otherArgs, tracePath,
+        GridMixRunMode.DATA_GENERATION_AND_RUN_GRIDMIX.getValue());
+  }
+  
+  /**
+   * Use existing uncompressed input data and verify the compression ratio 
+   * of reduce output against custom output compression ratio and also verify 
+   * the compression output file output format.
+   * @throws Exception -if an error occurs.
+   */
+  @Test
+  public void testCompressionEmulationOfCompressedOutputWithCustomRatios() 
+      throws Exception {
+    final String tracePath = getTraceFile("compression_case3_trace");
+    Assert.assertNotNull("Trace file has not found.", tracePath);
+    UtilsForGridmix.cleanup(gridmixDir, rtClient.getDaemonConf());
+    final String [] runtimeValues = { "LOADJOB",
+                                      SubmitterUserResolver.class.getName(),
+                                      "STRESS",
+                                      inputSizeInMB + "m",
+                                      tracePath };
+
+    final String [] otherArgs = { 
+        "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false",
+        "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=true",
+        "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false",
+        "-D", GridMixConfig.GRIDMIX_OUTPUT_COMPRESSION_RATIO + "=0.38"
+    };
+
+    runGridmixAndVerify(runtimeValues, otherArgs, tracePath, 
+        GridMixRunMode.DATA_GENERATION_AND_RUN_GRIDMIX.getValue());
+  }
+}

Added: hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestDisableGridmixEmulationOfHighRam.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestDisableGridmixEmulationOfHighRam.java?rev=1222572&view=auto
==============================================================================
--- hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestDisableGridmixEmulationOfHighRam.java (added)
+++ hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestDisableGridmixEmulationOfHighRam.java Fri Dec 23 05:38:53 2011
@@ -0,0 +1,65 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred.gridmix;
+
+import org.apache.commons.logging.LogFactory;
+import org.apache.commons.logging.Log;
+import org.apache.hadoop.mapred.gridmix.test.system.GridMixConfig;
+import org.apache.hadoop.mapred.gridmix.GridmixJob;
+import org.junit.Test;
+import org.junit.Assert;
+
+/**
+ * Run the {@link Gridmix} with a high ram jobs trace by disabling the 
+ * emulation of high ram  and verify each {@link Gridmix} job 
+ * whether it honors the high ram or not. In disable mode it should 
+ * should not honor the high ram and run it as a normal job.
+ */
+public class TestDisableGridmixEmulationOfHighRam 
+    extends GridmixSystemTestCase { 
+  private static final Log LOG = 
+      LogFactory.getLog("TestDisableGridmixEmulationOfHighRam.class");
+
+  /**
+   * Generate input data and run {@link Gridmix} with a high ram jobs trace 
+   * as a load job and STRESS submission policy in a SubmitterUserResolver 
+   * mode. Verify each {@link Gridmix} job whether it honors the 
+   * high ram or not after completion of execution. In disable mode the
+   * jobs should not honor the high ram. 
+   * @throws Exception - if an error occurs.
+   */
+  @Test
+  public void testEmulationOfHighRamForReducersOfMRJobs() 
+      throws Exception { 
+    final long inputSizeInMB = cSize * 250;
+    String tracePath = getTraceFile("highram_mr_jobs_case3");
+    Assert.assertNotNull("Trace file has not found.", tracePath);
+    String [] runtimeValues = {"LOADJOB",
+                               SubmitterUserResolver.class.getName(),
+                               "SERIAL",
+                               inputSizeInMB + "m",
+                               tracePath};
+
+    String [] otherArgs = {
+               "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false", 
+               "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false", 
+               "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false"};
+
+    runGridmixAndVerify(runtimeValues, otherArgs, tracePath);
+  }
+}

Added: hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestEmulationOfHDFSAndLocalFSDCFiles.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestEmulationOfHDFSAndLocalFSDCFiles.java?rev=1222572&view=auto
==============================================================================
--- hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestEmulationOfHDFSAndLocalFSDCFiles.java (added)
+++ hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestEmulationOfHDFSAndLocalFSDCFiles.java Fri Dec 23 05:38:53 2011
@@ -0,0 +1,95 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred.gridmix;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.mapred.gridmix.test.system.GridMixConfig;
+import org.apache.hadoop.mapred.gridmix.test.system.GridMixRunMode;
+import org.apache.hadoop.mapred.gridmix.test.system.UtilsForGridmix;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Verify the emulation of HDFS and Local FS distributed cache files against
+ * the given input trace file.
+ */
+public class TestEmulationOfHDFSAndLocalFSDCFiles extends 
+    GridmixSystemTestCase {
+  private static final Log LOG = 
+      LogFactory.getLog("TestEmulationOfLocalFSDCFiles.class");
+
+  /**
+   * Generate the input data and distributed cache files for HDFS and 
+   * local FS. Verify the gridmix emulation of HDFS and Local FS 
+   * distributed cache files in RoundRobinUserResolver mode with STRESS
+   * submission policy.
+   * @throws Exception - if an error occurs.
+   */
+  @Test
+  public void testGenerateDataEmulateHDFSAndLocalFSDCFiles() 
+     throws Exception  {
+    final long inputSizeInMB = 1024 * 6;
+    final String tracePath = getTraceFile("distcache_case8_trace");
+    Assert.assertNotNull("Trace file has not found.", tracePath);
+    final String [] runtimeValues = 
+                     {"LOADJOB",
+                      RoundRobinUserResolver.class.getName(),
+                      "STRESS",
+                      inputSizeInMB + "m",
+                      "file://" + UtilsForGridmix.getProxyUsersFile(conf),
+                      tracePath};
+
+    final String [] otherArgs = {
+       "-D", JobContext.JOB_CANCEL_DELEGATION_TOKEN + "=false",
+       "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=true",
+       "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false",
+       "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false"
+    };
+    runGridmixAndVerify(runtimeValues, otherArgs, tracePath, 
+        GridMixRunMode.DATA_GENERATION_AND_RUN_GRIDMIX.getValue());
+  }
+  
+  /**
+   * Use existing input and distributed cache files for HDFS and
+   * local FS. Verify the gridmix emulation of HDFS and Local FS
+   * distributed cache files in SubmitterUserResolver mode with REPLAY
+   * submission policy.
+   * @throws Exception - if an error occurs.
+   */
+  @Test
+  public void testEmulationOfHDFSAndLocalFSDCFiles() 
+     throws Exception  {
+    final String tracePath = getTraceFile("distcache_case8_trace");
+    Assert.assertNotNull("Trace file has not found.", tracePath);
+    final String [] runtimeValues ={"LOADJOB",
+                                    SubmitterUserResolver.class.getName(),
+                                    "STRESS",
+                                    tracePath};
+
+    final String [] otherArgs = { 
+       "-D", JobContext.JOB_CANCEL_DELEGATION_TOKEN + "=false",
+       "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=true",
+       "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false",
+       "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false"
+    };
+    runGridmixAndVerify(runtimeValues, otherArgs, tracePath, 
+                        GridMixRunMode.RUN_GRIDMIX.getValue());
+  }
+}

Added: hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestEmulationOfHDFSDCFileUsesMultipleJobs.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestEmulationOfHDFSDCFileUsesMultipleJobs.java?rev=1222572&view=auto
==============================================================================
--- hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestEmulationOfHDFSDCFileUsesMultipleJobs.java (added)
+++ hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestEmulationOfHDFSDCFileUsesMultipleJobs.java Fri Dec 23 05:38:53 2011
@@ -0,0 +1,91 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred.gridmix;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.mapred.gridmix.test.system.GridMixConfig;
+import org.apache.hadoop.mapred.gridmix.test.system.GridMixRunMode;
+import org.apache.hadoop.mapred.gridmix.test.system.UtilsForGridmix;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Verify the Gridmix emulation of HDFS distributed cache file which uses 
+ * different jobs that are submitted with different users.
+ */
+public class TestEmulationOfHDFSDCFileUsesMultipleJobs extends 
+    GridmixSystemTestCase {
+  private static final Log LOG = 
+      LogFactory.getLog("TestEmulationOfHDFSDCFileUsesMultipleJobs.class");
+
+  /**
+   * Generate the input data and HDFS distributed cache file based 
+   * on given input trace. Verify the Gridmix emulation of HDFS
+   * distributed cache file in RoundRobinResolver mode with 
+   * STRESS submission policy.
+   * @throws Exception - if an error occurs.
+   */
+  @Test
+  public void testGenerateAndEmulationOfHDFSDCFile() 
+     throws Exception { 
+    final long inputSizeInMB = 1024 * 6;
+    final String tracePath = getTraceFile("distcache_case9_trace");
+    Assert.assertNotNull("Trace file has not found.", tracePath);
+    final String [] runtimeValues = 
+                     {"LOADJOB",
+                      RoundRobinUserResolver.class.getName(),
+                      "STRESS",
+                      inputSizeInMB + "m",
+                      "file://" + UtilsForGridmix.getProxyUsersFile(conf),
+                      tracePath};
+
+    final String [] otherArgs = {
+        "-D", JobContext.JOB_CANCEL_DELEGATION_TOKEN + "=false",
+        "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false",
+        "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=true"
+    };
+    runGridmixAndVerify(runtimeValues, otherArgs, tracePath, 
+        GridMixRunMode.DATA_GENERATION_AND_RUN_GRIDMIX.getValue());
+  }
+  
+  /**
+   * Verify the Gridmix emulation of HDFS distributed cache
+   * file in SubmitterUserResolver mode with STRESS submission policy 
+   * by using the existing input data and HDFS distributed cache file. 
+   * @throws Exception - if an error occurs.
+   */
+  @Test
+  public void testGridmixEmulationOfHDFSPublicDCFile() 
+      throws Exception {
+    final String tracePath = getTraceFile("distcache_case9_trace");
+    Assert.assertNotNull("Trace file has not found.", tracePath);
+    final String [] runtimeValues = {"LOADJOB",
+                                     SubmitterUserResolver.class.getName(),
+                                     "STRESS",
+                                     tracePath};
+
+    final String [] otherArgs = {
+      "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false",
+      "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=true"
+    };
+    runGridmixAndVerify(runtimeValues, otherArgs, tracePath, 
+                        GridMixRunMode.RUN_GRIDMIX.getValue());
+  }
+}

Added: hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestEmulationOfHDFSDCFilesWithDifferentVisibilities.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestEmulationOfHDFSDCFilesWithDifferentVisibilities.java?rev=1222572&view=auto
==============================================================================
--- hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestEmulationOfHDFSDCFilesWithDifferentVisibilities.java (added)
+++ hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestEmulationOfHDFSDCFilesWithDifferentVisibilities.java Fri Dec 23 05:38:53 2011
@@ -0,0 +1,92 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred.gridmix;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.mapred.gridmix.test.system.GridMixConfig;
+import org.apache.hadoop.mapred.gridmix.test.system.GridMixRunMode;
+import org.apache.hadoop.mapred.gridmix.test.system.UtilsForGridmix;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ *  Verify the Gridmix emulation of HDFS distributed cache files of 
+ *  different visibilities. 
+ */
+
+public class TestEmulationOfHDFSDCFilesWithDifferentVisibilities 
+    extends GridmixSystemTestCase {
+  private static final Log LOG = 
+     LogFactory.getLog(
+         "TestEmulationOfHDFSDCFilesWithDifferentVisibilities.class");
+  
+  /**
+   * Generate input data and HDFS distributed cache files of different
+   * visibilities based on given input trace. Verify the Gridmix emulation 
+   * of HDFS distributed cache files of different visibilities in 
+   * RoundRobinUserResolver mode with SERIAL submission policy.
+   * @throws Exception - if an error occurs.
+   */
+  @Test
+  public void testGenerateAndEmulateOfHDFSDCFilesWithDiffVisibilities() 
+     throws Exception {
+    final long INPUT_SIZE = 1024 * 9;
+    final String tracePath = getTraceFile("distcache_case5_trace");
+    Assert.assertNotNull("Trace file was not found.", tracePath);
+    final String [] runtimeValues = 
+                     { "LOADJOB",
+                       RoundRobinUserResolver.class.getName(),
+                       "STRESS",
+                       INPUT_SIZE+"m",
+                       "file://" + UtilsForGridmix.getProxyUsersFile(conf),
+                       tracePath};
+
+    final String [] otherArgs = { 
+        "-D", JobContext.JOB_CANCEL_DELEGATION_TOKEN + "=false",
+        "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false",
+        "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE +  "=true"
+    };
+    runGridmixAndVerify(runtimeValues, otherArgs, tracePath, 
+        GridMixRunMode.DATA_GENERATION_AND_RUN_GRIDMIX.getValue());
+  }
+  
+  /**
+   * Disable the distributed cache emulation and verify the Gridmix jobs
+   * whether it emulates or not. 
+   * @throws Exception
+   */
+  @Test
+  public void testHDFSDCFilesWithoutEnableDCEmulation() 
+     throws Exception {
+    final String tracePath = getTraceFile("distcache_case6_trace");
+    Assert.assertNotNull("Trace file was not found.", tracePath);
+    final String [] runtimeValues ={ "LOADJOB",
+                                     SubmitterUserResolver.class.getName(),
+                                     "STRESS",
+                                     tracePath};
+    final String [] otherArgs = {
+        "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false",
+        "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false"
+    };
+    runGridmixAndVerify(runtimeValues, otherArgs, tracePath, 
+                        GridMixRunMode.RUN_GRIDMIX.getValue());
+  }
+}
+



Mime
View raw message