hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From omal...@apache.org
Subject svn commit: r1079263 - in /hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system: org/apache/hadoop/mapred/gridmix/test/system/ resources/
Date Tue, 08 Mar 2011 06:01:43 GMT
Author: omalley
Date: Tue Mar  8 06:01:43 2011
New Revision: 1079263

URL: http://svn.apache.org/viewvc?rev=1079263&view=rev
Log:
commit e67c34006222669a2e4cb35176cf28aed949c245
Author: Vinay Kumar Thota <vinayt@yahoo-inc.com>
Date:   Thu Feb 10 04:39:11 2011 -0800

    3964674 from

Added:
    hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/resources/
    hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/resources/trace_10m.json.gz
  (with props)
    hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/resources/trace_12m.json.gz
  (with props)
    hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/resources/trace_1m.json.gz
  (with props)
    hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/resources/trace_3m.json.gz
  (with props)
    hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/resources/trace_5m.json.gz
  (with props)
    hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/resources/trace_7m.json.gz
  (with props)
Modified:
    hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/test/system/UtilsForGridmix.java

Modified: hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/test/system/UtilsForGridmix.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/test/system/UtilsForGridmix.java?rev=1079263&r1=1079262&r2=1079263&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/test/system/UtilsForGridmix.java
(original)
+++ hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/test/system/UtilsForGridmix.java
Tue Mar  8 06:01:43 2011
@@ -28,7 +28,7 @@ import org.apache.hadoop.util.ToolRunner
 import org.apache.hadoop.mapred.gridmix.Gridmix;
 import org.apache.hadoop.conf.Configuration;
 import java.util.Date;
-import java.util.Hashtable;
+import java.util.HashMap;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.net.URI;
@@ -48,6 +48,9 @@ import org.apache.hadoop.test.system.Pro
  */
 public class UtilsForGridmix {
   private static final Log LOG = LogFactory.getLog(UtilsForGridmix.class);
+  private static final Path DEFAULT_TRACES_PATH =
+    new Path(System.getProperty("user.dir") +
+    "/src/test/system/resources/");
 
   /**
    * cleanup the folder or file.
@@ -320,4 +323,139 @@ public class UtilsForGridmix {
      }
      return pud;
   }
+
+  /**
+   *  Gives the list of paths for MR traces against different time 
+   *  intervals.It fetches only the paths which followed the below 
+   *  file convention.
+   *    Syntax : &lt;FileName&gt;_&lt;TimeIntervals&gt;.json.gz
+   *  There is a restriction in a  file and user has to  
+   *  follow  the below convention for time interval.
+   *    Syntax: &lt;numeric&gt;[m|h|d] 
+   *    e.g : for 10 minutes trace should specify 10m, 
+   *    same way for 1 hour traces should specify 1h, 
+   *    for 1 day traces should specify 1d.
+   *
+   * @param conf - cluster configuration.
+   * @return - list of MR paths as key/value pair based on time interval.
+   * @throws IOException - if an I/O error occurs.
+   */
+  public static Map<String, String> getMRTraces(Configuration conf) 
+     throws IOException {
+    return getMRTraces(conf, DEFAULT_TRACES_PATH);
+  }
+  
+  /**
+   *  It gives the list of paths for MR traces against different time 
+   *  intervals. It fetches only the paths which followed the below 
+   *  file convention.
+   *    Syntax : &lt;FileNames&gt;_&lt;TimeInterval&gt;.json.gz
+   *  There is a restriction in a file and user has to follow the 
+   *  below convention for time interval. 
+   *    Syntax: &lt;numeric&gt;[m|h|d] 
+   *    e.g : for 10 minutes trace should specify 10m,
+   *    same way for 1 hour traces should specify 1h, 
+   *    for 1 day  traces should specify 1d.
+   *
+   * @param conf - cluster configuration object.
+   * @param tracesPath - MR traces path.
+   * @return - list of MR paths as key/value pair based on time interval.
+   * @throws IOException - If an I/O error occurs.
+   */
+  public static Map<String,String> getMRTraces(Configuration conf, 
+      Path tracesPath) throws IOException {
+     Map <String, String> jobTraces = new HashMap <String, String>();
+     final FileSystem fs = FileSystem.getLocal(conf);
+     final FileStatus fstat[] = fs.listStatus(tracesPath);
+     for (FileStatus fst : fstat) {
+        final String fileName = fst.getPath().getName();
+        if (fileName.endsWith("m.json.gz") || 
+            fileName.endsWith("h.json.gz") || 
+            fileName.endsWith("d.json.gz")) {
+           jobTraces.put(fileName.substring(fileName.indexOf("_") + 1, 
+              fileName.indexOf(".json.gz")), fst.getPath().toString());
+        }
+     }
+     if (jobTraces.size() == 0) {
+        LOG.error("No traces found in " + tracesPath.toString() + " path.");
+        throw new IOException("No traces found in " + 
+            tracesPath.toString() + " path.");
+     }
+     return jobTraces;
+  }
+  
+  /**
+   * It list the all the MR traces path irrespective of time.
+   * @param conf - cluster configuration.
+   * @param tracesPath - MR traces path
+   * @return - MR paths as a list.
+   * @throws IOException - if an I/O error occurs.
+   */
+  public static List<String> listMRTraces(Configuration conf, 
+      Path tracesPath) throws IOException {
+     List<String> jobTraces = new ArrayList<String>();
+     final FileSystem fs = FileSystem.getLocal(conf);
+     final FileStatus fstat[] = fs.listStatus(tracesPath);
+     for (FileStatus fst : fstat) {
+        jobTraces.add(fst.getPath().toString());
+     }
+     if (jobTraces.size() == 0) {
+        LOG.error("No traces found in " + tracesPath.toString() + " path.");
+        throw new IOException("No traces found in " + 
+            tracesPath.toString() + " path.");
+     }
+     return jobTraces;
+  }
+  
+  /**
+   * It list the all the MR traces path irrespective of time.
+   * @param conf - cluster configuration.
+   * @param tracesPath - MR traces path
+   * @return - MR paths as a list.
+   * @throws IOException - if an I/O error occurs.
+   */
+  public static List<String> listMRTraces(Configuration conf) 
+      throws IOException {
+     return listMRTraces(conf, DEFAULT_TRACES_PATH);
+  }
+
+  /**
+   * Gives the list of MR traces for given time interval.
+   * The time interval should be following convention.
+   *   Syntax : &lt;numeric&gt;[m|h|d]
+   *   e.g : 10m or 1h or 2d etc.
+   * @param conf - cluster configuration
+   * @param timeInterval - trace time interval.
+   * @param tracesPath - MR traces Path.
+   * @return - MR paths as a list for a given time interval.
+   * @throws IOException - If an I/O error occurs.
+   */
+  public static List<String> listMRTracesByTime(Configuration conf, 
+      String timeInterval, Path tracesPath) throws IOException {
+     List<String> jobTraces = new ArrayList<String>();
+     final FileSystem fs = FileSystem.getLocal(conf);
+     final FileStatus fstat[] = fs.listStatus(tracesPath);
+     for (FileStatus fst : fstat) {
+        final String fileName = fst.getPath().getName();
+        if (fileName.indexOf(timeInterval) >= 0) {
+           jobTraces.add(fst.getPath().toString());
+        }
+     }
+     return jobTraces;
+  }
+  
+  /**
+   * Gives the list of MR traces for given time interval.
+   * The time interval should be following convention.
+   *   Syntax : &lt;numeric&gt;[m|h|d]
+   *   e.g : 10m or 1h or 2d etc.
+   * @param conf - cluster configuration
+   * @param timeInterval - trace time interval.
+   * @return - MR paths as a list for a given time interval.
+   * @throws IOException - If an I/O error occurs.
+   */
+  public static List<String> listMRTracesByTime(Configuration conf, 
+      String timeInterval) throws IOException {
+     return listMRTracesByTime(conf, timeInterval, DEFAULT_TRACES_PATH);
+  }
 }

Added: hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/resources/trace_10m.json.gz
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/resources/trace_10m.json.gz?rev=1079263&view=auto
==============================================================================
Binary file - no diff available.

Propchange: hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/resources/trace_10m.json.gz
------------------------------------------------------------------------------
    svn:mime-type = application/octet-stream

Added: hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/resources/trace_12m.json.gz
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/resources/trace_12m.json.gz?rev=1079263&view=auto
==============================================================================
Binary file - no diff available.

Propchange: hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/resources/trace_12m.json.gz
------------------------------------------------------------------------------
    svn:mime-type = application/octet-stream

Added: hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/resources/trace_1m.json.gz
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/resources/trace_1m.json.gz?rev=1079263&view=auto
==============================================================================
Binary file - no diff available.

Propchange: hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/resources/trace_1m.json.gz
------------------------------------------------------------------------------
    svn:mime-type = application/octet-stream

Added: hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/resources/trace_3m.json.gz
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/resources/trace_3m.json.gz?rev=1079263&view=auto
==============================================================================
Binary file - no diff available.

Propchange: hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/resources/trace_3m.json.gz
------------------------------------------------------------------------------
    svn:mime-type = application/octet-stream

Added: hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/resources/trace_5m.json.gz
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/resources/trace_5m.json.gz?rev=1079263&view=auto
==============================================================================
Binary file - no diff available.

Propchange: hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/resources/trace_5m.json.gz
------------------------------------------------------------------------------
    svn:mime-type = application/octet-stream

Added: hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/resources/trace_7m.json.gz
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/resources/trace_7m.json.gz?rev=1079263&view=auto
==============================================================================
Binary file - no diff available.

Propchange: hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/resources/trace_7m.json.gz
------------------------------------------------------------------------------
    svn:mime-type = application/octet-stream



Mime
View raw message