hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From maha...@apache.org
Subject svn commit: r931743 - in /hadoop/mapreduce/trunk: ./ src/test/mapred/org/apache/hadoop/tools/rumen/ src/test/tools/data/rumen/small-trace-test/ src/test/tools/data/rumen/small-trace-test/counters-format-test-logs/ src/tools/org/apache/hadoop/tools/rumen/
Date Thu, 08 Apr 2010 00:58:03 GMT
Author: mahadev
Date: Thu Apr  8 00:58:02 2010
New Revision: 931743

URL: http://svn.apache.org/viewvc?rev=931743&view=rev
Log:
MAPREDUCE-1523. Sometimes rumen trace generator fails to extract the job finish time. (dick
king via mahadev)

Added:
    hadoop/mapreduce/trunk/src/test/tools/data/rumen/small-trace-test/counters-format-test-logs/
    hadoop/mapreduce/trunk/src/test/tools/data/rumen/small-trace-test/counters-format-test-logs/megacluster.megacorp.com_1265616107882_job_201002080801_40864_conf.xml
    hadoop/mapreduce/trunk/src/test/tools/data/rumen/small-trace-test/counters-format-test-logs/megacluster.megacorp.com_1265616107882_job_201002080801_40864_job_name-DAILY%2F20100210%5D.gz
  (with props)
    hadoop/mapreduce/trunk/src/test/tools/data/rumen/small-trace-test/counters-format-test-logs/megacluster.megacorp.com_1265616107882_job_201002080801_50510_conf.xml
    hadoop/mapreduce/trunk/src/test/tools/data/rumen/small-trace-test/counters-format-test-logs/megacluster.megacorp.com_1265616107882_job_201002080801_50510_job_name-DAILY%2F20100208%5D.gz
  (with props)
    hadoop/mapreduce/trunk/src/test/tools/data/rumen/small-trace-test/counters-test-trace.json.gz
  (with props)
Modified:
    hadoop/mapreduce/trunk/CHANGES.txt
    hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/tools/rumen/TestRumenJobTraces.java
    hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/HistoryEventEmitter.java
    hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/Job20LineHistoryEventEmitter.java
    hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/MapAttempt20LineHistoryEventEmitter.java
    hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/ReduceAttempt20LineHistoryEventEmitter.java
    hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/Task20LineHistoryEventEmitter.java
    hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/TaskAttempt20LineEventEmitter.java
    hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/TraceBuilder.java

Modified: hadoop/mapreduce/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/CHANGES.txt?rev=931743&r1=931742&r2=931743&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/CHANGES.txt (original)
+++ hadoop/mapreduce/trunk/CHANGES.txt Thu Apr  8 00:58:02 2010
@@ -519,6 +519,9 @@ Trunk (unreleased changes)
     MAPREDUCE-1585. Create Hadoop Archives version 2 with filenames
     URL-encoded (rodrigo via mahadev)
 
+    MAPREDUCE-1523. Sometimes rumen trace generator fails to extract the job
+    finish time. (dick king via mahadev)
+
 Release 0.21.0 - Unreleased
 
   INCOMPATIBLE CHANGES

Modified: hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/tools/rumen/TestRumenJobTraces.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/tools/rumen/TestRumenJobTraces.java?rev=931743&r1=931742&r2=931743&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/tools/rumen/TestRumenJobTraces.java
(original)
+++ hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/tools/rumen/TestRumenJobTraces.java
Thu Apr  8 00:58:02 2010
@@ -137,7 +137,7 @@ public class TestRumenJobTraces {
         new Path(rootInputPath, "dispatch-sample-v20-jt-log.gz");
 
     System.out.println("topology result file = " + topologyPath);
-    System.out.println("trace result file = " + tracePath);
+    System.out.println("testRumenViaDispatch() trace result file = " + tracePath);
 
     String demuxerClassName = ConcatenatedInputFilesDemuxer.class.getName();
 
@@ -162,6 +162,44 @@ public class TestRumenJobTraces {
   }
 
   @Test
+  public void testBracketedCounters() throws Exception {
+    final Configuration conf = new Configuration();
+    final FileSystem lfs = FileSystem.getLocal(conf);
+
+    final Path rootInputDir =
+        new Path(System.getProperty("test.tools.input.dir", "")).makeQualified(
+            lfs.getUri(), lfs.getWorkingDirectory());
+    final Path rootTempDir =
+        new Path(System.getProperty("test.build.data", "/tmp")).makeQualified(
+            lfs.getUri(), lfs.getWorkingDirectory());
+
+    final Path rootInputPath = new Path(rootInputDir, "rumen/small-trace-test");
+    final Path tempDir = new Path(rootTempDir, "TestBracketedCounters");
+    lfs.delete(tempDir, true);
+
+    final Path topologyPath = new Path(tempDir, "dispatch-topology.json");
+    final Path tracePath = new Path(tempDir, "dispatch-trace.json");
+
+    final Path inputPath = new Path(rootInputPath, "counters-format-test-logs");
+
+    System.out.println("topology result file = " + topologyPath);
+    System.out.println("testBracketedCounters() trace result file = " + tracePath);
+
+    final Path goldPath =
+        new Path(rootInputPath, "counters-test-trace.json.gz");
+
+    String[] args =
+        { tracePath.toString(), topologyPath.toString(), inputPath.toString() };
+
+    Tool analyzer = new TraceBuilder();
+    int result = ToolRunner.run(analyzer, args);
+    assertEquals("Non-zero exit", 0, result);
+
+    TestRumenJobTraces.<LoggedJob> jsonFileMatchesGold(conf, tracePath,
+        goldPath, LoggedJob.class, "trace");
+  }
+
+  @Test
   public void testHadoop20JHParser() throws Exception {
     final Configuration conf = new Configuration();
     final FileSystem lfs = FileSystem.getLocal(conf);

Added: hadoop/mapreduce/trunk/src/test/tools/data/rumen/small-trace-test/counters-format-test-logs/megacluster.megacorp.com_1265616107882_job_201002080801_40864_conf.xml
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/tools/data/rumen/small-trace-test/counters-format-test-logs/megacluster.megacorp.com_1265616107882_job_201002080801_40864_conf.xml?rev=931743&view=auto
==============================================================================
--- hadoop/mapreduce/trunk/src/test/tools/data/rumen/small-trace-test/counters-format-test-logs/megacluster.megacorp.com_1265616107882_job_201002080801_40864_conf.xml
(added)
+++ hadoop/mapreduce/trunk/src/test/tools/data/rumen/small-trace-test/counters-format-test-logs/megacluster.megacorp.com_1265616107882_job_201002080801_40864_conf.xml
Thu Apr  8 00:58:02 2010
@@ -0,0 +1,3 @@
+<configuration>
+<property><name>mapred.child.java.opts</name><value>-Xmx1024M -Djava.io.tmpdir=./tmp</value></property>
+</configuration>

Added: hadoop/mapreduce/trunk/src/test/tools/data/rumen/small-trace-test/counters-format-test-logs/megacluster.megacorp.com_1265616107882_job_201002080801_40864_job_name-DAILY%2F20100210%5D.gz
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/tools/data/rumen/small-trace-test/counters-format-test-logs/megacluster.megacorp.com_1265616107882_job_201002080801_40864_job_name-DAILY%252F20100210%255D.gz?rev=931743&view=auto
==============================================================================
Binary file - no diff available.

Propchange: hadoop/mapreduce/trunk/src/test/tools/data/rumen/small-trace-test/counters-format-test-logs/megacluster.megacorp.com_1265616107882_job_201002080801_40864_job_name-DAILY%2F20100210%5D.gz
------------------------------------------------------------------------------
    svn:mime-type = application/octet-stream

Added: hadoop/mapreduce/trunk/src/test/tools/data/rumen/small-trace-test/counters-format-test-logs/megacluster.megacorp.com_1265616107882_job_201002080801_50510_conf.xml
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/tools/data/rumen/small-trace-test/counters-format-test-logs/megacluster.megacorp.com_1265616107882_job_201002080801_50510_conf.xml?rev=931743&view=auto
==============================================================================
--- hadoop/mapreduce/trunk/src/test/tools/data/rumen/small-trace-test/counters-format-test-logs/megacluster.megacorp.com_1265616107882_job_201002080801_50510_conf.xml
(added)
+++ hadoop/mapreduce/trunk/src/test/tools/data/rumen/small-trace-test/counters-format-test-logs/megacluster.megacorp.com_1265616107882_job_201002080801_50510_conf.xml
Thu Apr  8 00:58:02 2010
@@ -0,0 +1,3 @@
+<configuration>
+<property><name>mapred.child.java.opts</name><value>-Xmx1024m -Djava.net.preferIPv4Stack=true</value></property>
+</configuration>

Added: hadoop/mapreduce/trunk/src/test/tools/data/rumen/small-trace-test/counters-format-test-logs/megacluster.megacorp.com_1265616107882_job_201002080801_50510_job_name-DAILY%2F20100208%5D.gz
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/tools/data/rumen/small-trace-test/counters-format-test-logs/megacluster.megacorp.com_1265616107882_job_201002080801_50510_job_name-DAILY%252F20100208%255D.gz?rev=931743&view=auto
==============================================================================
Binary file - no diff available.

Propchange: hadoop/mapreduce/trunk/src/test/tools/data/rumen/small-trace-test/counters-format-test-logs/megacluster.megacorp.com_1265616107882_job_201002080801_50510_job_name-DAILY%2F20100208%5D.gz
------------------------------------------------------------------------------
    svn:mime-type = application/octet-stream

Added: hadoop/mapreduce/trunk/src/test/tools/data/rumen/small-trace-test/counters-test-trace.json.gz
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/tools/data/rumen/small-trace-test/counters-test-trace.json.gz?rev=931743&view=auto
==============================================================================
Binary file - no diff available.

Propchange: hadoop/mapreduce/trunk/src/test/tools/data/rumen/small-trace-test/counters-test-trace.json.gz
------------------------------------------------------------------------------
    svn:mime-type = application/octet-stream

Modified: hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/HistoryEventEmitter.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/HistoryEventEmitter.java?rev=931743&r1=931742&r2=931743&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/HistoryEventEmitter.java
(original)
+++ hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/HistoryEventEmitter.java
Thu Apr  8 00:58:02 2010
@@ -63,6 +63,15 @@ abstract class HistoryEventEmitter {
     return new Pair<Queue<HistoryEvent>, PostEmitAction>(results, removeEmitter);
   }
 
+  protected static Counters maybeParseCounters(String counters) {
+    try {
+      return parseCounters(counters);
+    } catch (ParseException e) {
+      LOG.warn("The counter string, \"" + counters + "\" is badly formatted.");
+      return null;
+    }
+  }
+
   protected static Counters parseCounters(String counters)
       throws ParseException {
     if (counters == null) {
@@ -73,6 +82,8 @@ abstract class HistoryEventEmitter {
     counters = counters.replace("\\.", "\\\\.");
     counters = counters.replace("\\\\(", "\\(");
     counters = counters.replace("\\\\)", "\\)");
+    counters = counters.replace("\\\\[", "\\[");
+    counters = counters.replace("\\\\]", "\\]");
 
     org.apache.hadoop.mapred.Counters depForm =
         org.apache.hadoop.mapred.Counters.fromEscapedCompactString(counters);

Modified: hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/Job20LineHistoryEventEmitter.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/Job20LineHistoryEventEmitter.java?rev=931743&r1=931742&r2=931743&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/Job20LineHistoryEventEmitter.java
(original)
+++ hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/Job20LineHistoryEventEmitter.java
Thu Apr  8 00:58:02 2010
@@ -226,15 +226,10 @@ public class Job20LineHistoryEventEmitte
       if (status != null && status.equalsIgnoreCase("success")
           && finishTime != null && finishedMaps != null
           && finishedReduces != null) {
-        try {
-          return new JobFinishedEvent(jobID, Long.parseLong(finishTime),
-              Integer.parseInt(finishedMaps),
-              Integer.parseInt(finishedReduces), Integer.parseInt(failedMaps),
-              Integer.parseInt(failedReduces), null, null,
-              parseCounters(counters));
-        } catch (ParseException e) {
-          return null;
-        }
+        return new JobFinishedEvent(jobID, Long.parseLong(finishTime), Integer
+            .parseInt(finishedMaps), Integer.parseInt(finishedReduces), Integer
+            .parseInt(failedMaps), Integer.parseInt(failedReduces), null, null,
+            maybeParseCounters(counters));
       }
 
       return null;

Modified: hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/MapAttempt20LineHistoryEventEmitter.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/MapAttempt20LineHistoryEventEmitter.java?rev=931743&r1=931742&r2=931743&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/MapAttempt20LineHistoryEventEmitter.java
(original)
+++ hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/MapAttempt20LineHistoryEventEmitter.java
Thu Apr  8 00:58:02 2010
@@ -60,23 +60,18 @@ public class MapAttempt20LineHistoryEven
 
       if (finishTime != null && status != null
           && status.equalsIgnoreCase("success")) {
-
-        try {
-          String hostName = line.get("HOSTNAME");
-          String counters = line.get("COUNTERS");
-          String state = line.get("STATE_STRING");
-
-          MapAttempt20LineHistoryEventEmitter that =
-              (MapAttempt20LineHistoryEventEmitter) thatg;
-
-          if (finishTime != null && "success".equalsIgnoreCase(status)) {
-            return new MapAttemptFinishedEvent(taskAttemptID,
-                that.originalTaskType, status, Long.parseLong(finishTime), Long
-                    .parseLong(finishTime), hostName, state,
-                parseCounters(counters));
-          }
-        } catch (ParseException e) {
-          // no code
+        String hostName = line.get("HOSTNAME");
+        String counters = line.get("COUNTERS");
+        String state = line.get("STATE_STRING");
+
+        MapAttempt20LineHistoryEventEmitter that =
+            (MapAttempt20LineHistoryEventEmitter) thatg;
+
+        if (finishTime != null && "success".equalsIgnoreCase(status)) {
+          return new MapAttemptFinishedEvent(taskAttemptID,
+              that.originalTaskType, status, Long.parseLong(finishTime), Long
+                  .parseLong(finishTime), hostName, state,
+              maybeParseCounters(counters));
         }
       }
 

Modified: hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/ReduceAttempt20LineHistoryEventEmitter.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/ReduceAttempt20LineHistoryEventEmitter.java?rev=931743&r1=931742&r2=931743&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/ReduceAttempt20LineHistoryEventEmitter.java
(original)
+++ hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/ReduceAttempt20LineHistoryEventEmitter.java
Thu Apr  8 00:58:02 2010
@@ -60,26 +60,21 @@ public class ReduceAttempt20LineHistoryE
 
       if (finishTime != null && status != null
           && status.equalsIgnoreCase("success")) {
-
-        try {
-          String hostName = line.get("HOSTNAME");
-          String counters = line.get("COUNTERS");
-          String state = line.get("STATE_STRING");
-          String shuffleFinish = line.get("SHUFFLE_FINISHED");
-          String sortFinish = line.get("SORT_FINISHED");
-
-          if (finishTime != null && shuffleFinish != null && sortFinish !=
null
-              && "success".equalsIgnoreCase(status)) {
-            ReduceAttempt20LineHistoryEventEmitter that =
-                (ReduceAttempt20LineHistoryEventEmitter) thatg;
-
-            return new ReduceAttemptFinishedEvent(taskAttemptID,
-                that.originalTaskType, status, Long.parseLong(shuffleFinish),
-                Long.parseLong(sortFinish), Long.parseLong(finishTime),
-                hostName, state, parseCounters(counters));
-          }
-        } catch (ParseException e) {
-          // no code
+        String hostName = line.get("HOSTNAME");
+        String counters = line.get("COUNTERS");
+        String state = line.get("STATE_STRING");
+        String shuffleFinish = line.get("SHUFFLE_FINISHED");
+        String sortFinish = line.get("SORT_FINISHED");
+
+        if (finishTime != null && shuffleFinish != null && sortFinish !=
null
+            && "success".equalsIgnoreCase(status)) {
+          ReduceAttempt20LineHistoryEventEmitter that =
+              (ReduceAttempt20LineHistoryEventEmitter) thatg;
+
+          return new ReduceAttemptFinishedEvent(taskAttemptID,
+              that.originalTaskType, status, Long.parseLong(shuffleFinish),
+              Long.parseLong(sortFinish), Long.parseLong(finishTime), hostName,
+              state, maybeParseCounters(counters));
         }
       }
 

Modified: hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/Task20LineHistoryEventEmitter.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/Task20LineHistoryEventEmitter.java?rev=931743&r1=931742&r2=931743&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/Task20LineHistoryEventEmitter.java
(original)
+++ hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/Task20LineHistoryEventEmitter.java
Thu Apr  8 00:58:02 2010
@@ -119,13 +119,7 @@ public class Task20LineHistoryEventEmitt
 
       if (finishTime != null && error == null
           && (status != null && status.equalsIgnoreCase("success"))) {
-        Counters eventCounters = null;
-
-        try {
-          eventCounters = parseCounters(counters);
-        } catch (ParseException e) {
-          // no code
-        }
+        Counters eventCounters = maybeParseCounters(counters);
 
         Task20LineHistoryEventEmitter that =
             (Task20LineHistoryEventEmitter) thatg;

Modified: hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/TaskAttempt20LineEventEmitter.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/TaskAttempt20LineEventEmitter.java?rev=931743&r1=931742&r2=931743&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/TaskAttempt20LineEventEmitter.java
(original)
+++ hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/TaskAttempt20LineEventEmitter.java
Thu Apr  8 00:58:02 2010
@@ -101,21 +101,16 @@ public abstract class TaskAttempt20LineE
 
       if (finishTime != null && status != null
           && status.equalsIgnoreCase("success")) {
+        String hostName = line.get("HOSTNAME");
+        String counters = line.get("COUNTERS");
+        String state = line.get("STATE_STRING");
 
-        try {
-          String hostName = line.get("HOSTNAME");
-          String counters = line.get("COUNTERS");
-          String state = line.get("STATE_STRING");
-
-          TaskAttempt20LineEventEmitter that =
-              (TaskAttempt20LineEventEmitter) thatg;
-
-          return new TaskAttemptFinishedEvent(taskAttemptID,
-              that.originalTaskType, status, Long.parseLong(finishTime),
-              hostName, state, parseCounters(counters));
-        } catch (ParseException e) {
-          return null;
-        }
+        TaskAttempt20LineEventEmitter that =
+            (TaskAttempt20LineEventEmitter) thatg;
+
+        return new TaskAttemptFinishedEvent(taskAttemptID,
+            that.originalTaskType, status, Long.parseLong(finishTime),
+            hostName, state, maybeParseCounters(counters));
       }
 
       return null;

Modified: hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/TraceBuilder.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/TraceBuilder.java?rev=931743&r1=931742&r2=931743&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/TraceBuilder.java (original)
+++ hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/TraceBuilder.java Thu Apr
 8 00:58:02 2010
@@ -103,11 +103,11 @@ public class TraceBuilder extends Config
 
           List<String> dirNames = new ArrayList<String>();
 
-          for (int j = 0; j < statuses.length; ++j) {
-            String name = statuses[j].getPath().getName();
+          for (FileStatus s : statuses) {
+            if (s.isDir()) continue;
+            String name = s.getPath().getName();
 
-            if (!(name.length() >= 4 && ".crc".equals(name.substring(name
-                .length() - 4)))) {
+            if (!(name.endsWith(".crc") || name.startsWith("."))) {
               dirNames.add(name);
             }
           }
@@ -126,12 +126,14 @@ public class TraceBuilder extends Config
     }
   }
 
-  public static void main(String[] args) throws Exception {
+  public static void main(String[] args) {
     TraceBuilder builder = new TraceBuilder();
     int result = RUN_METHOD_FAILED_EXIT_CODE;
 
     try {
-      result = ToolRunner.run(builder, args);
+      result = ToolRunner.run(builder, args); 
+    } catch (Throwable t) {
+      t.printStackTrace(System.err);
     } finally {
       try {
         builder.finish();
@@ -199,61 +201,65 @@ public class TraceBuilder extends Config
       for (Path p : options.inputs) {
         InputDemuxer inputDemuxer = options.inputDemuxerClass.newInstance();
 
-        inputDemuxer.bindTo(p, getConf());
+        try {
+          inputDemuxer.bindTo(p, getConf());
+        } catch (IOException e) {
+          LOG.warn("Unable to bind Path " + p + " .  Skipping...", e);
 
-        if (inputDemuxer != null) {
-          Pair<String, InputStream> filePair = null;
+          continue;
+        }
 
-          try {
-            while ((filePair = inputDemuxer.getNext()) != null) {
-              RewindableInputStream ris =
-                  new RewindableInputStream(filePair.second());
-
-              JobHistoryParser parser = null;
-
-              try {
-                String jobID = extractJobID(filePair.first());
-                if (jobID == null) {
-                  LOG.warn("File skipped: Invalid file name: "
-                      + filePair.first());
-                  continue;
-                }
-                if ((jobBuilder == null)
-                    || (!jobBuilder.getJobID().equals(jobID))) {
-                  if (jobBuilder != null) {
-                    traceWriter.output(jobBuilder.build());
-                  }
-                  jobBuilder = new JobBuilder(jobID);
-                }
+        Pair<String, InputStream> filePair = null;
 
-                if (isJobConfXml(filePair.first(), ris)) {
-                  processJobConf(jobConfParser.parse(ris.rewind()), jobBuilder);
-                } else {
-                  parser = JobHistoryParserFactory.getParser(ris);
-                  if (parser == null) {
-                    LOG.warn("File skipped: Cannot find suitable parser: "
-                        + filePair.first());
-                  } else {
-                    processJobHistory(parser, jobBuilder);
-                  }
+        try {
+          while ((filePair = inputDemuxer.getNext()) != null) {
+            RewindableInputStream ris =
+                new RewindableInputStream(filePair.second());
+
+            JobHistoryParser parser = null;
+
+            try {
+              String jobID = extractJobID(filePair.first());
+              if (jobID == null) {
+                LOG.warn("File skipped: Invalid file name: "
+                    + filePair.first());
+                continue;
+              }
+              if ((jobBuilder == null)
+                  || (!jobBuilder.getJobID().equals(jobID))) {
+                if (jobBuilder != null) {
+                  traceWriter.output(jobBuilder.build());
                 }
-              } finally {
+                jobBuilder = new JobBuilder(jobID);
+              }
+
+              if (isJobConfXml(filePair.first(), ris)) {
+                processJobConf(jobConfParser.parse(ris.rewind()), jobBuilder);
+              } else {
+                parser = JobHistoryParserFactory.getParser(ris);
                 if (parser == null) {
-                  ris.close();
+                  LOG.warn("File skipped: Cannot find suitable parser: "
+                      + filePair.first());
                 } else {
-                  parser.close();
-                  parser = null;
+                  processJobHistory(parser, jobBuilder);
                 }
               }
+            } finally {
+              if (parser == null) {
+                ris.close();
+              } else {
+                parser.close();
+                parser = null;
+              }
             }
-          } catch (Throwable t) {
-            if (filePair != null) {
-              LOG.warn("TraceBuilder got an error while processing file "
-                  + filePair.first(), t);
-            }
-          } finally {
-            inputDemuxer.close();
           }
+        } catch (Throwable t) {
+          if (filePair != null) {
+            LOG.warn("TraceBuilder got an error while processing the [possibly virtual] file
"
+                + filePair.first() + " within Path " + p , t);
+          }
+        } finally {
+          inputDemuxer.close();
         }
       }
       if (jobBuilder != null) {



Mime
View raw message