hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From omal...@apache.org
Subject svn commit: r1079262 - in /hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix: TestGridMixDataGeneration.java TestGridMixFilePool.java test/system/GridMixConfig.java test/system/UtilsForGridmix.java
Date Tue, 08 Mar 2011 06:01:33 GMT
Author: omalley
Date: Tue Mar  8 06:01:33 2011
New Revision: 1079262

URL: http://svn.apache.org/viewvc?rev=1079262&view=rev
Log:
commit df3ff80d8f32cc14edae564d7a2b82ea6cbf3139
Author: Vinay Kumar Thota <vinayt@yahoo-inc.com>
Date:   Tue Feb 8 01:13:34 2011 -0800

    3957917 from
    
    +++ b/YAHOO-CHANGES.txt
    +
    +   [MR-2053] : Test Gridmix file pool for different input
    +  file sizes based on pool minimum size.Patch available at
    +   (vinayt)

Added:
    hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridMixFilePool.java
Modified:
    hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridMixDataGeneration.java
    hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/test/system/GridMixConfig.java
    hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/test/system/UtilsForGridmix.java

Modified: hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridMixDataGeneration.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridMixDataGeneration.java?rev=1079262&r1=1079261&r2=1079262&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridMixDataGeneration.java
(original)
+++ hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridMixDataGeneration.java
Tue Mar  8 06:01:33 2011
@@ -91,8 +91,14 @@ public class TestGridMixDataGeneration {
        inputSize+"m",
        "file:///dev/null"}; 
 
+    String [] otherArgs = {
+      "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE +
+      "=false",
+      "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE +
+      "=false"
+    };
     int exitCode = UtilsForGridmix.runGridmixJob(gridmixDir, 
-      conf,GridMixRunMode.DATA_GENERATION, runtimeValues);
+      conf,GridMixRunMode.DATA_GENERATION, runtimeValues, otherArgs);
     Assert.assertEquals("Data generation has failed.", 0 , exitCode);
     checkGeneratedDataAndJobStatus(inputSize);
   }
@@ -114,8 +120,15 @@ public class TestGridMixDataGeneration {
        "file://" + UtilsForGridmix.getProxyUsersFile(conf),
        "file:///dev/null"};
     
+    String [] otherArgs = {
+      "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE +
+      "=false",
+      "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE +
+      "=false"
+    };
+
     int exitCode = UtilsForGridmix.runGridmixJob(gridmixDir, 
-       conf,GridMixRunMode.DATA_GENERATION, runtimeValues);
+       conf,GridMixRunMode.DATA_GENERATION, runtimeValues, otherArgs);
     Assert.assertEquals("Data generation has failed.", 0 , exitCode);
     checkGeneratedDataAndJobStatus(inputSize); 
   }
@@ -140,7 +153,11 @@ public class TestGridMixDataGeneration {
     int bytesPerFile = 200; // 200 mb per file of data
     String [] otherArgs = {
       "-D", GridMixConfig.GRIDMIX_BYTES_PER_FILE + 
-      "=" + (bytesPerFile * 1024 * 1024)
+      "=" + (bytesPerFile * 1024 * 1024),
+      "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE +
+      "=false",
+      "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE +
+      "=false"
     };
     int exitCode = UtilsForGridmix.runGridmixJob(gridmixDir, 
        conf,GridMixRunMode.DATA_GENERATION, runtimeValues,otherArgs);

Added: hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridMixFilePool.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridMixFilePool.java?rev=1079262&view=auto
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridMixFilePool.java
(added)
+++ hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridMixFilePool.java
Tue Mar  8 06:01:33 2011
@@ -0,0 +1,124 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred.gridmix;
+import org.apache.commons.logging.LogFactory;
+import org.apache.commons.logging.Log;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.mapreduce.test.system.MRCluster;
+import org.apache.hadoop.mapreduce.test.system.JTClient;
+import org.apache.hadoop.mapreduce.test.system.JTProtocol;
+import org.apache.hadoop.mapred.gridmix.FilePool;
+import org.apache.hadoop.mapred.gridmix.test.system.UtilsForGridmix;
+import org.apache.hadoop.mapred.gridmix.test.system.GridMixRunMode;
+import org.apache.hadoop.mapred.gridmix.test.system.GridMixConfig;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.FileStatus;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.AfterClass;
+import org.junit.Test;
+import java.io.IOException;
+import java.util.ArrayList;
+
+public class TestGridMixFilePool {
+  private static final Log LOG = LogFactory
+     .getLog(TestGridMixFilePool.class);
+  private static Configuration conf = new Configuration();
+  private static MRCluster cluster;
+  private static JTProtocol remoteClient;
+  private static JTClient jtClient;
+  private static Path gridmixDir;
+  private static int clusterSize; 
+  
+  @BeforeClass
+  public static void before() throws Exception {
+    String []  excludeExpList = {"java.net.ConnectException", 
+       "java.io.IOException"};
+    cluster = MRCluster.createCluster(conf);
+    cluster.setExcludeExpList(excludeExpList);
+    cluster.setUp();
+    jtClient = cluster.getJTClient();
+    remoteClient = jtClient.getProxy();
+    clusterSize = cluster.getTTClients().size();
+    gridmixDir = new Path("herriot-gridmix");
+    UtilsForGridmix.createDirs(gridmixDir, remoteClient.getDaemonConf());
+  }
+
+  @AfterClass
+  public static void after() throws Exception {
+    UtilsForGridmix.cleanup(gridmixDir, conf);
+    cluster.tearDown();
+  }
+  
+  @Test
+  public void testFilesCountAndSizesForSpecifiedFilePool() throws Exception {
+    conf = remoteClient.getDaemonConf();
+    final long inputSize = clusterSize * 200;
+    int [] fileSizesInMB = {50, 100, 400, 50, 300, 10, 60, 40, 20 ,10 , 500};
+    long targetSize = Long.MAX_VALUE;
+    final int expFileCount = clusterSize + 4;
+    String [] runtimeValues ={"LOADJOB",
+       SubmitterUserResolver.class.getName(),
+       "STRESS",
+       inputSize+"m",
+       "file:///dev/null"}; 
+
+    String [] otherArgs = {
+      "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE +
+      "=false",
+      "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE +
+      "=false"
+    };
+
+    int exitCode = UtilsForGridmix.runGridmixJob(gridmixDir, 
+       conf,GridMixRunMode.DATA_GENERATION, runtimeValues, otherArgs);
+    Assert.assertEquals("Data generation has failed.", 0 , exitCode);
+    // create files for given sizes.
+    createFiles(new Path(gridmixDir,"input"),fileSizesInMB);
+    conf.setLong(FilePool.GRIDMIX_MIN_FILE, 100 * 1024 * 1024);
+    FilePool fpool = new FilePool(conf,new Path(gridmixDir,"input"));
+    fpool.refresh();
+    verifyFilesSizeAndCountForSpecifiedPool(expFileCount,targetSize, fpool);
+  }
+  
+  private void createFiles(Path inputDir, int [] fileSizes) 
+     throws Exception {
+    for (int size : fileSizes) {
+      UtilsForGridmix.createFile(size, inputDir, conf);
+    }
+  }
+  
+  private void verifyFilesSizeAndCountForSpecifiedPool(int expFileCount, 
+     long minFileSize, FilePool pool) throws IOException {
+    final ArrayList<FileStatus> files = new ArrayList<FileStatus>();
+    long  actFilesSize = pool.getInputFiles(minFileSize, files)/(1024 * 1024);
+    long expFilesSize = (clusterSize * 200) + 1300;
+    Assert.assertEquals("Files Size has not matched for specified pool.",
+       expFilesSize, actFilesSize);
+    int actFileCount = files.size();    
+    Assert.assertEquals("File count has not matched.", 
+       expFileCount, actFileCount);
+    int count = 0;
+    for (FileStatus fstat : files) {
+      String fp = fstat.getPath().toString();
+      count = count + ((fp.indexOf("datafile_") > 0)? 0 : 1);
+    }
+    Assert.assertEquals("Total folders are not matched with cluster size", 
+            clusterSize, count);
+  }
+}

Modified: hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/test/system/GridMixConfig.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/test/system/GridMixConfig.java?rev=1079262&r1=1079261&r2=1079262&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/test/system/GridMixConfig.java
(original)
+++ hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/test/system/GridMixConfig.java
Tue Mar  8 06:01:33 2011
@@ -72,4 +72,16 @@ public class GridMixConfig {
    */
   public static final String GRIDMIX_MINIMUM_FILE_SIZE =
       "gridmix.min.file.size";
+
+  /**
+   * Gridmix compression enable
+   */
+  public static final String GRIDMIX_COMPRESSION_ENABLE =
+      "gridmix.compression-emulation.enable";
+
+  /**
+   * Gridmix distcache enable
+   */
+  public static final String GRIDMIX_DISTCACHE_ENABLE = 
+      "gridmix.distributed-cache-emulation.enable";
 }

Modified: hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/test/system/UtilsForGridmix.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/test/system/UtilsForGridmix.java?rev=1079262&r1=1079261&r2=1079262&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/test/system/UtilsForGridmix.java
(original)
+++ hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/test/system/UtilsForGridmix.java
Tue Mar  8 06:01:33 2011
@@ -86,7 +86,7 @@ public class UtilsForGridmix {
        "-D", GridMixConfig.GRIDMIX_LOG_MODE + 
        "=DEBUG",
        "-D", GridMixConfig.GRIDMIX_OUTPUT_DIR + 
-       "=" + new Path(gridmixDir,"gridmix").toString(),
+       "=" + new Path("gridmix").toString(),
        "-D", GridMixConfig.GRIDMIX_JOB_SUBMISSION_QUEUE_IN_TRACE 
        + "=true",
        "-D", GridMixConfig.GRIDMIX_JOB_TYPE 
@@ -103,22 +103,23 @@ public class UtilsForGridmix {
      classArgs = new String[]{
         "-generate", values[3], 
         "-users", values[4], 
-        new Path(gridmixDir,"input").toString(), 
+        gridmixDir.toString(), 
         values[5]};
     } else if (gridmixRunMode == GridMixRunMode.DATA_GENERATION ||
        gridmixRunMode == GridMixRunMode.DATA_GENERATION_AND_RUN_GRIDMIX){
       classArgs = new String[]{
-         "-generate", values[3], new Path(gridmixDir,"input").toString(),
+         "-generate", values[3], 
+         gridmixDir.toString(),
          values[4]};
     } else if(gridmixRunMode == GridMixRunMode.RUN_GRIDMIX 
        && values[1].indexOf("RoundRobinUserResolver") > 0) {
       classArgs = new String[]{         
          "-users", values[3], 
-         new Path(gridmixDir,"input").toString(),
+         gridmixDir.toString(),
          values[4]};
     } else {
       classArgs = new String[]{
-         new Path(gridmixDir,"input").toString(),values[3]};
+         gridmixDir.toString(),values[3]};
     }
     
     String [] args = new String [runtimeArgs.length + 



Mime
View raw message