hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cdoug...@apache.org
Subject svn commit: r813140 - in /hadoop/mapreduce/trunk: CHANGES.txt src/test/mapred/org/apache/hadoop/tools/rumen/TestRumenJobTraces.java src/tools/org/apache/hadoop/tools/rumen/HadoopLogsAnalyzer.java
Date Wed, 09 Sep 2009 22:22:52 GMT
Author: cdouglas
Date: Wed Sep  9 22:22:52 2009
New Revision: 813140

URL: http://svn.apache.org/viewvc?rev=813140&view=rev
Log:
MAPREDUCE-930. Modify Rumen to resolve paths in the canonical way, rather
than defaulting to the local filesystem.

Modified:
    hadoop/mapreduce/trunk/CHANGES.txt
    hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/tools/rumen/TestRumenJobTraces.java
    hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/HadoopLogsAnalyzer.java

Modified: hadoop/mapreduce/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/CHANGES.txt?rev=813140&r1=813139&r2=813140&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/CHANGES.txt (original)
+++ hadoop/mapreduce/trunk/CHANGES.txt Wed Sep  9 22:22:52 2009
@@ -318,6 +318,9 @@
     MAPREDUCE-960. Remove an unnecessary intermediate copy and obsolete API
     from KeyValueLineRecordReader. (cdouglas)
 
+    MAPREDUCE-930. Modify Rumen to resolve paths in the canonical way, rather
+    than defaulting to the local filesystem. (cdouglas)
+
   BUG FIXES
 
     MAPREDUCE-878. Rename fair scheduler design doc to 

Modified: hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/tools/rumen/TestRumenJobTraces.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/tools/rumen/TestRumenJobTraces.java?rev=813140&r1=813139&r2=813140&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/tools/rumen/TestRumenJobTraces.java
(original)
+++ hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/tools/rumen/TestRumenJobTraces.java
Wed Sep  9 22:22:52 2009
@@ -25,9 +25,15 @@
 import java.io.FileInputStream;
 import java.io.FileOutputStream;
 import java.io.IOException;
+import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.io.PrintStream;
 
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.util.ToolRunner;
+
 import org.codehaus.jackson.JsonParseException;
 import org.codehaus.jackson.JsonParser;
 import org.codehaus.jackson.JsonProcessingException;
@@ -38,94 +44,80 @@
 import junit.framework.TestCase;
 
 public class TestRumenJobTraces extends TestCase {
-  public void testSmallTrace() throws IOException {
-    File tempDirectory = new File(System.getProperty("test.build.data", "/tmp"));
 
-    String rootInputDir = System.getProperty("test.tools.input.dir", "");
-    String rootTempDir = System.getProperty("test.build.data", "");
+  public void testSmallTrace() throws Exception {
+    final Configuration conf = new Configuration();
+    final FileSystem lfs = FileSystem.getLocal(conf);
 
-    File rootInputFile = new File(new File(rootInputDir),
-        "rumen/small-trace-test");
-    File tempDirFile = new File(rootTempDir);
+    final Path rootInputDir = new Path(
+        System.getProperty("test.tools.input.dir", "")).makeQualified(lfs);
+    final Path rootTempDir = new Path(
+        System.getProperty("test.build.data", "/tmp")).makeQualified(lfs);
 
-    assertFalse("property test.build.data is not defined", ""
-        .equals(rootTempDir));
-    assertFalse("property test.tools.input.dir is not defined", ""
-        .equals(rootInputDir));
 
-    if (rootInputDir.charAt(rootInputDir.length() - 1) == '/') {
-      rootInputDir = rootInputDir.substring(0, rootInputDir.length() - 1);
-    }
+    final Path rootInputFile = new Path(rootInputDir, "rumen/small-trace-test");
+    final Path tempDir = new Path(rootTempDir, "TestRumenJobTraces");
+    lfs.delete(tempDir, true);
 
-    if (rootTempDir.charAt(rootTempDir.length() - 1) == '/') {
-      rootTempDir = rootTempDir.substring(0, rootTempDir.length() - 1);
-    }
+    assertFalse("property test.build.data is not defined",
+        "".equals(rootTempDir));
+    assertFalse("property test.tools.input.dir is not defined",
+        "".equals(rootInputDir));
 
-    File topologyFile = File.createTempFile("topology", ".json", tempDirFile);
-    File traceFile = File.createTempFile("trace", ".json", tempDirFile);
+    final Path topologyFile = new Path(tempDir, "topology.json");
+    final Path traceFile = new Path(tempDir, "trace.json");
 
-    File inputFile = new File(rootInputFile, "sample-job-tracker-logs");
+    final Path inputFile = new Path(rootInputFile, "sample-job-tracker-logs");
 
-    // topologyFile.deleteOnExit();
-    // traceFile.deleteOnExit();
-    System.out.println("topology result file = "
-        + topologyFile.getCanonicalPath());
-    System.out.println("trace result file = " + traceFile.getCanonicalPath());
+    System.out.println("topology result file = " + topologyFile);
+    System.out.println("trace result file = " + traceFile);
 
     String[] args = new String[6];
 
     args[0] = "-v1";
 
     args[1] = "-write-topology";
-    args[2] = topologyFile.getPath();
+    args[2] = topologyFile.toString();
 
     args[3] = "-write-job-trace";
-    args[4] = traceFile.getPath();
-
-    args[5] = inputFile.getPath();
+    args[4] = traceFile.toString();
 
-    assertTrue("The input file " + inputFile.getPath() + " does not exist.",
-        inputFile.canRead());
-    assertTrue("The output topology file " + topologyFile.getPath()
-        + " cannot be written.", topologyFile.canWrite());
-    assertTrue("The output trace file " + traceFile.getPath()
-        + " cannot be written.", traceFile.canWrite());
+    args[5] = inputFile.toString();
 
     PrintStream old_stdout = System.out;
 
-    File stdoutFile = File.createTempFile("stdout", ".text", tempDirFile);
+    final Path stdoutFile = new Path(tempDir, "stdout.text");
 
-    // stdoutFile.deleteOnExit();
-    System.out.println("stdout file = " + stdoutFile.getCanonicalPath());
+    System.out.println("stdout file = " + stdoutFile);
 
     PrintStream enveloped_stdout = new PrintStream(new BufferedOutputStream(
-        new FileOutputStream(stdoutFile)));
+          lfs.create(stdoutFile, true)));
 
-    File topologyGoldFile = new File(rootInputFile,
+    final Path topologyGoldFile = new Path(rootInputFile, 
         "job-tracker-logs-topology-output");
-    File traceGoldFile = new File(rootInputFile,
+    final Path traceGoldFile = new Path(rootInputFile,
         "job-tracker-logs-trace-output");
 
     try {
       System.setOut(enveloped_stdout);
 
-      HadoopLogsAnalyzer.main(args);
+      HadoopLogsAnalyzer analyzer = new HadoopLogsAnalyzer();
+
+      int result = ToolRunner.run(analyzer, args);
 
       enveloped_stdout.close();
+
+      assertEquals("Non-zero exit", 0, result);
+
     } finally {
       System.setOut(old_stdout);
     }
 
-    jsonFileMatchesGold(topologyFile, topologyGoldFile,
+    jsonFileMatchesGold(lfs, topologyFile, topologyGoldFile,
         new LoggedNetworkTopology(), "topology");
-    jsonFileMatchesGold(traceFile, traceGoldFile, new LoggedJob(), "trace");
-
-    System.out
-        .println("These files have been erased because the tests have succeeded.");
+    jsonFileMatchesGold(lfs, traceFile, traceGoldFile, new LoggedJob(),
+        "trace");
 
-    topologyFile.deleteOnExit();
-    traceFile.deleteOnExit();
-    stdoutFile.deleteOnExit();
   }
 
   /*
@@ -182,7 +174,7 @@
    * @throws IOException
    */
   private void statisticalTest(String args[], String inputFname,
-      String goldFilename, boolean inputIsDirectory) throws IOException {
+      String goldFilename, boolean inputIsDirectory) throws Exception {
     File tempDirectory = new File(System.getProperty("test.build.data", "/tmp"));
 
     String rootInputDir = System.getProperty("test.tools.input.dir", "");
@@ -232,13 +224,16 @@
     try {
       System.setOut(enveloped_stdout);
 
-      HadoopLogsAnalyzer.main(newArgs);
+      HadoopLogsAnalyzer analyzer = new HadoopLogsAnalyzer();
+
+      int result = ToolRunner.run(analyzer, args);
 
       enveloped_stdout.close();
 
       System.setOut(old_stdout);
 
       assertFilesMatch(stdoutFile, jobDistroGold);
+      assertEquals("Non-zero exit", 0, result);
     } finally {
       System.setOut(old_stdout);
     }
@@ -284,13 +279,13 @@
     assertFalse("Line number " + currentLineNumber + " disagrees", true);
   }
 
-  static private void jsonFileMatchesGold(File result, File gold, Object obj,
-      String fileDescription) throws IOException {
-    FileInputStream goldStream = new FileInputStream(gold);
+  static private void jsonFileMatchesGold(FileSystem lfs, Path result,
+        Path gold, Object obj, String fileDescription) throws IOException {
+    InputStream goldStream = lfs.open(gold);
     BufferedReader goldReader = new BufferedReader(new InputStreamReader(
         goldStream));
 
-    FileInputStream resultStream = new FileInputStream(result);
+    InputStream resultStream = lfs.open(result);
     BufferedReader resultReader = new BufferedReader(new InputStreamReader(
         resultStream));
 

Modified: hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/HadoopLogsAnalyzer.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/HadoopLogsAnalyzer.java?rev=813140&r1=813139&r2=813140&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/HadoopLogsAnalyzer.java
(original)
+++ hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/HadoopLogsAnalyzer.java
Wed Sep  9 22:22:52 2009
@@ -125,8 +125,6 @@
    */
   private boolean omitTaskDetails = false;
 
-  private String jobTraceFilename = null;
-
   private JsonGenerator jobTraceGen = null;
 
   private boolean prettyprintTrace = true;
@@ -149,8 +147,6 @@
 
   private int[] attemptTimesPercentiles;
 
-  private String topologyFilename = null;
-
   private JsonGenerator topologyGen = null;
 
   private HashSet<ParsedHost> allHosts = new HashSet<ParsedHost>();
@@ -305,17 +301,6 @@
     super();
   }
 
-  private Path parsePathString(String pathname) {
-    Path wd = null;
-    try {
-      wd = FileSystem.getLocal(getConf()).getWorkingDirectory();
-    } catch (IOException e) {
-      return new Path(pathname);
-    }
-
-    return new Path(wd, pathname);
-  }
-
   private boolean pathIsDirectory(Path p) throws IOException {
     FileSystem fs = p.getFileSystem(getConf());
     return fs.getFileStatus(p).isDir();
@@ -329,6 +314,8 @@
    */
   private int initializeHadoopLogsAnalyzer(String[] args)
       throws FileNotFoundException, IOException {
+    Path jobTraceFilename = null;
+    Path topologyFilename = null;
     if (args.length == 0 || args[args.length - 1].charAt(0) == '-') {
       inputFilename = null;
     } else {
@@ -351,7 +338,7 @@
       // these control the job digest
       if ("-write-job-trace".equals(args[i].toLowerCase())) {
         ++i;
-        jobTraceFilename = args[i];
+        jobTraceFilename = new Path(args[i]);
         continue;
       }
 
@@ -367,7 +354,7 @@
 
       if ("-write-topology".equals(args[i].toLowerCase())) {
         ++i;
-        topologyFilename = args[i];
+        topologyFilename = new Path(args[i]);
         continue;
       }
 
@@ -462,7 +449,7 @@
     taskMapAttemptFinishTimes = new HashMap<String, Long>();
     taskReduceAttemptFinishTimes = new HashMap<String, Long>();
 
-    Path inputPath = parsePathString(inputFilename);
+    final Path inputPath = new Path(inputFilename);
 
     inputIsDirectory = pathIsDirectory(inputPath);
 
@@ -518,10 +505,9 @@
       jmapper.configure(
           SerializationConfig.Feature.CAN_OVERRIDE_ACCESS_MODIFIERS, true);
       JsonFactory jfactory = jmapper.getJsonFactory();
-      FileSystem jobFS = parsePathString(jobTraceFilename).getFileSystem(
-          getConf());
-      jobTraceGen = jfactory.createJsonGenerator(jobFS
-          .create(parsePathString(jobTraceFilename)), JsonEncoding.UTF8);
+      FileSystem jobFS = jobTraceFilename.getFileSystem(getConf());
+      jobTraceGen = jfactory.createJsonGenerator(
+          jobFS.create(jobTraceFilename), JsonEncoding.UTF8);
       if (prettyprintTrace) {
         jobTraceGen.useDefaultPrettyPrinter();
       }
@@ -531,10 +517,9 @@
         tmapper.configure(
             SerializationConfig.Feature.CAN_OVERRIDE_ACCESS_MODIFIERS, true);
         JsonFactory tfactory = tmapper.getJsonFactory();
-        FileSystem topoFS = parsePathString(topologyFilename).getFileSystem(
-            getConf());
-        topologyGen = tfactory.createJsonGenerator(topoFS
-            .create(parsePathString(topologyFilename)), JsonEncoding.UTF8);
+        FileSystem topoFS = topologyFilename.getFileSystem(getConf());
+        topologyGen = tfactory.createJsonGenerator(
+            topoFS.create(topologyFilename), JsonEncoding.UTF8);
         topologyGen.useDefaultPrettyPrinter();
       }
     }



Mime
View raw message