hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ama...@apache.org
Subject svn commit: r1130554 - in /hadoop/mapreduce/trunk: CHANGES.txt src/contrib/gridmix/src/test/org/apache/hadoop/mapred/gridmix/TestHighRamJob.java
Date Thu, 02 Jun 2011 13:56:01 GMT
Author: amarrk
Date: Thu Jun  2 13:56:00 2011
New Revision: 1130554

URL: http://svn.apache.org/viewvc?rev=1130554&view=rev
Log:
MAPREDUCE-2543. [Gridmix] High-Ram feature emulation testcase. (amarrk)

Added:
    hadoop/mapreduce/trunk/src/contrib/gridmix/src/test/org/apache/hadoop/mapred/gridmix/TestHighRamJob.java
Modified:
    hadoop/mapreduce/trunk/CHANGES.txt

Modified: hadoop/mapreduce/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/CHANGES.txt?rev=1130554&r1=1130553&r2=1130554&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/CHANGES.txt (original)
+++ hadoop/mapreduce/trunk/CHANGES.txt Thu Jun  2 13:56:00 2011
@@ -25,6 +25,8 @@ Trunk (unreleased changes)
 
   IMPROVEMENTS
 
+    MAPREDUCE-2543. [Gridmix] High-Ram feature emulation testcase. (amarrk)
+
     MAPREDUCE-2469. Task counters should also report the total heap usage of 
     the task. (Ravi Gummadi and Amar Ramesh Kamat via amarrk)
 

Added: hadoop/mapreduce/trunk/src/contrib/gridmix/src/test/org/apache/hadoop/mapred/gridmix/TestHighRamJob.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/gridmix/src/test/org/apache/hadoop/mapred/gridmix/TestHighRamJob.java?rev=1130554&view=auto
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/gridmix/src/test/org/apache/hadoop/mapred/gridmix/TestHighRamJob.java
(added)
+++ hadoop/mapreduce/trunk/src/contrib/gridmix/src/test/org/apache/hadoop/mapred/gridmix/TestHighRamJob.java
Thu Jun  2 13:56:00 2011
@@ -0,0 +1,195 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred.gridmix;
+
+import static org.junit.Assert.*;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.gridmix.DebugJobProducer.MockJob;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.MRConfig;
+import org.apache.hadoop.mapreduce.MRJobConfig;
+import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.tools.rumen.JobStory;
+import org.junit.Test;
+
+/**
+ * Test if Gridmix correctly configures the simulated job's configuration for
+ * high ram job properties.
+ */
+public class TestHighRamJob {
+  /**
+   * A dummy {@link GridmixJob} that opens up the simulated job for testing.
+   */
+  private static class DummyGridmixJob extends GridmixJob {
+    public DummyGridmixJob(Configuration conf, JobStory desc) 
+    throws IOException {
+      super(conf, System.currentTimeMillis(), desc, new Path("test"), 
+            UserGroupInformation.getCurrentUser(), -1);
+    }
+    
+    /**
+     * Do nothing since this is a dummy gridmix job.
+     */
+    @Override
+    public Job call() throws Exception {
+      return null;
+    }
+    
+    @Override
+    protected boolean canEmulateCompression() {
+      // return false as we don't need compression
+      return false;
+    }
+    
+    protected Job getJob() {
+      // open the simulated job for testing
+      return job;
+    }
+  }
+  
+  private static void testHighRamConfig(long jobMapMB, long jobReduceMB, 
+      long clusterMapMB, long clusterReduceMB, long simulatedClusterMapMB, 
+      long simulatedClusterReduceMB, long expectedMapMB, long expectedReduceMB, 
+      Configuration gConf) 
+  throws IOException {
+    Configuration simulatedJobConf = new Configuration(gConf);
+    simulatedJobConf.setLong(MRConfig.MAPMEMORY_MB, simulatedClusterMapMB);
+    simulatedJobConf.setLong(MRConfig.REDUCEMEMORY_MB, 
+                             simulatedClusterReduceMB);
+    
+    // define a source conf
+    Configuration sourceConf = new Configuration();
+    
+    // configure the original job
+    sourceConf.setLong(MRJobConfig.MAP_MEMORY_MB, jobMapMB);
+    sourceConf.setLong(MRConfig.MAPMEMORY_MB, clusterMapMB);
+    sourceConf.setLong(MRJobConfig.REDUCE_MEMORY_MB, jobReduceMB);
+    sourceConf.setLong(MRConfig.REDUCEMEMORY_MB, clusterReduceMB);
+    
+    // define a mock job
+    MockJob story = new MockJob(sourceConf);
+    
+    GridmixJob job = new DummyGridmixJob(simulatedJobConf, story);
+    Job simulatedJob = job.getJob();
+    Configuration simulatedConf = simulatedJob.getConfiguration();
+    
+    // check if the high ram properties are not set
+    assertEquals(expectedMapMB, 
+                 simulatedConf.getLong(MRJobConfig.MAP_MEMORY_MB,
+                                       JobConf.DISABLED_MEMORY_LIMIT));
+    assertEquals(expectedReduceMB, 
+                 simulatedConf.getLong(MRJobConfig.REDUCE_MEMORY_MB, 
+                                       JobConf.DISABLED_MEMORY_LIMIT));
+  }
+  
+  /**
+   * Tests high ram job properties configuration.
+   */
+  @SuppressWarnings("deprecation")
+  @Test
+  public void testHighRamFeatureEmulation() throws IOException {
+    // define the gridmix conf
+    Configuration gridmixConf = new Configuration();
+    
+    // test : check high ram emulation disabled
+    gridmixConf.setBoolean(GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE, false);
+    testHighRamConfig(10, 20, 5, 10, JobConf.DISABLED_MEMORY_LIMIT, 
+                      JobConf.DISABLED_MEMORY_LIMIT, 
+                      JobConf.DISABLED_MEMORY_LIMIT, 
+                      JobConf.DISABLED_MEMORY_LIMIT, gridmixConf);
+    
+    // test : check with high ram enabled (default) and no scaling
+    gridmixConf = new Configuration();
+    // set the deprecated max memory limit
+    gridmixConf.setLong(JobConf.UPPER_LIMIT_ON_TASK_VMEM_PROPERTY, 
+                        20*1024*1024);
+    testHighRamConfig(10, 20, 5, 10, 5, 10, 10, 20, gridmixConf);
+    
+    // test : check with high ram enabled and scaling
+    gridmixConf = new Configuration();
+    // set the new max map/reduce memory limits
+    gridmixConf.setLong(JTConfig.JT_MAX_MAPMEMORY_MB, 100);
+    gridmixConf.setLong(JTConfig.JT_MAX_REDUCEMEMORY_MB, 300);
+    testHighRamConfig(10, 45, 5, 15, 50, 100, 100, 300, gridmixConf);
+    
+    // test : check with high ram enabled and map memory scaling mismatch 
+    //        (deprecated)
+    gridmixConf = new Configuration();
+    gridmixConf.setLong(JobConf.UPPER_LIMIT_ON_TASK_VMEM_PROPERTY, 
+                        70*1024*1024);
+    Boolean failed = null;
+    try {
+      testHighRamConfig(10, 45, 5, 15, 50, 100, 100, 300, gridmixConf);
+      failed = false;
+    } catch (Exception e) {
+      failed = true;
+    }
+    assertNotNull(failed);
+    assertTrue("Exception expected for exceeding map memory limit "
+               + "(deprecation)!", failed);
+    
+    // test : check with high ram enabled and reduce memory scaling mismatch 
+    //        (deprecated)
+    gridmixConf = new Configuration();
+    gridmixConf.setLong(JobConf.UPPER_LIMIT_ON_TASK_VMEM_PROPERTY, 
+                        150*1024*1024);
+    failed = null;
+    try {
+      testHighRamConfig(10, 45, 5, 15, 50, 100, 100, 300, gridmixConf);
+      failed = false;
+    } catch (Exception e) {
+      failed = true;
+    }
+    assertNotNull(failed);
+    assertTrue("Exception expected for exceeding reduce memory limit "
+               + "(deprecation)!", failed);
+    
+    // test : check with high ram enabled and scaling mismatch on map limits
+    gridmixConf = new Configuration();
+    gridmixConf.setLong(JTConfig.JT_MAX_MAPMEMORY_MB, 70);
+    failed = null;
+    try {
+      testHighRamConfig(10, 45, 5, 15, 50, 100, 100, 300, gridmixConf);
+      failed = false;
+    } catch (Exception e) {
+      failed = true;
+    }
+    assertNotNull(failed);
+    assertTrue("Exception expected for exceeding map memory limit!", failed);
+    
+    // test : check with high ram enabled and scaling mismatch on reduce 
+    //        limits
+    gridmixConf = new Configuration();
+    gridmixConf.setLong(JTConfig.JT_MAX_REDUCEMEMORY_MB, 200);
+    failed = null;
+    try {
+      testHighRamConfig(10, 45, 5, 15, 50, 100, 100, 300, gridmixConf);
+      failed = false;
+    } catch (Exception e) {
+      failed = true;
+    }
+    assertNotNull(failed);
+    assertTrue("Exception expected for exceeding reduce memory limit!", failed);
+  }
+}
\ No newline at end of file



Mime
View raw message