hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From acmur...@apache.org
Subject svn commit: r592857 - in /lucene/hadoop/trunk: ./ src/contrib/streaming/src/java/org/apache/hadoop/streaming/ src/contrib/streaming/src/test/org/apache/hadoop/streaming/
Date Wed, 07 Nov 2007 19:12:42 GMT
Author: acmurthy
Date: Wed Nov  7 11:12:40 2007
New Revision: 592857

URL: http://svn.apache.org/viewvc?rev=592857&view=rev
Log:
HADOOP-1952.  Improve handling of invalid, user-specified classes while configuring streaming
jobs such as combiner, input/output formats etc. Now invalid options are caught, logged and
jobs are failed early. Contributed by Lohit Vijayarenu.

Modified:
    lucene/hadoop/trunk/CHANGES.txt
    lucene/hadoop/trunk/src/contrib/streaming/src/java/org/apache/hadoop/streaming/StreamJob.java
    lucene/hadoop/trunk/src/contrib/streaming/src/test/org/apache/hadoop/streaming/TestGzipInput.java
    lucene/hadoop/trunk/src/contrib/streaming/src/test/org/apache/hadoop/streaming/TestStreaming.java
    lucene/hadoop/trunk/src/contrib/streaming/src/test/org/apache/hadoop/streaming/TestStreamingFailure.java

Modified: lucene/hadoop/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/CHANGES.txt?rev=592857&r1=592856&r2=592857&view=diff
==============================================================================
--- lucene/hadoop/trunk/CHANGES.txt (original)
+++ lucene/hadoop/trunk/CHANGES.txt Wed Nov  7 11:12:40 2007
@@ -79,6 +79,11 @@
     HADOOP-1348.  Allow XML comments inside configuration files. 
     (Rajagopal Natarajan and Enis Soztutar via enis)
 
+    HADOOP-1952.  Improve handling of invalid, user-specified classes while
+    configuring streaming jobs such as combiner, input/output formats etc.
+    Now invalid options are caught, logged and jobs are failed early. (Lohit
+    Vijayarenu via acmurthy)
+
 Release 0.15.1 -
 
   BUG FIXES

Modified: lucene/hadoop/trunk/src/contrib/streaming/src/java/org/apache/hadoop/streaming/StreamJob.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/streaming/src/java/org/apache/hadoop/streaming/StreamJob.java?rev=592857&r1=592856&r2=592857&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/contrib/streaming/src/java/org/apache/hadoop/streaming/StreamJob.java
(original)
+++ lucene/hadoop/trunk/src/contrib/streaming/src/java/org/apache/hadoop/streaming/StreamJob.java
Wed Nov  7 11:12:40 2007
@@ -693,28 +693,33 @@
       fmt = TextInputFormat.class;
     } else if (inputFormatSpec_ != null) {
       if (inputFormatSpec_.equals(TextInputFormat.class.getName())
-          || inputFormatSpec_.equals(TextInputFormat.class.getCanonicalName())) {
+          || inputFormatSpec_.equals(TextInputFormat.class.getCanonicalName())
+          || inputFormatSpec_.equals(TextInputFormat.class.getSimpleName())) {
         fmt = TextInputFormat.class;
       } else if (inputFormatSpec_.equals(KeyValueTextInputFormat.class
           .getName())
           || inputFormatSpec_.equals(KeyValueTextInputFormat.class
-              .getCanonicalName())) {
+              .getCanonicalName())
+          || inputFormatSpec_.equals(KeyValueTextInputFormat.class.getSimpleName())) {
       } else if (inputFormatSpec_.equals(SequenceFileInputFormat.class
           .getName())
           || inputFormatSpec_
               .equals(org.apache.hadoop.mapred.SequenceFileInputFormat.class
-                  .getCanonicalName())) {
+                  .getCanonicalName())
+          || inputFormatSpec_
+              .equals(org.apache.hadoop.mapred.SequenceFileInputFormat.class.getSimpleName()))
{
       } else if (inputFormatSpec_.equals(SequenceFileAsTextInputFormat.class
           .getName())
           || inputFormatSpec_.equals(SequenceFileAsTextInputFormat.class
-              .getCanonicalName())) {
+              .getCanonicalName())
+          || inputFormatSpec_.equals(SequenceFileAsTextInputFormat.class.getSimpleName()))
{
         fmt = SequenceFileAsTextInputFormat.class;
       } else {
         c = StreamUtil.goodClassOrNull(inputFormatSpec_, defaultPackage);
         if (c != null) {
           fmt = c;
         } else {
-
+          fail("-inputformat : class not found : " + inputFormatSpec_);
         }
       }
     } 
@@ -744,7 +749,9 @@
       c = StreamUtil.goodClassOrNull(comCmd_, defaultPackage);
       if (c != null) {
         jobConf_.setCombinerClass(c);
-      } 
+      } else {
+        fail("-combiner : class not found : " + comCmd_);
+      }
     }
 
     boolean reducerNone_ = false;
@@ -791,7 +798,9 @@
       c = StreamUtil.goodClassOrNull(outputFormatSpec_, defaultPackage);
       if (c != null) {
         fmt = c;
-      } 
+      } else {
+        fail("-outputformat : class not found : " + outputFormatSpec_);
+      }
     }
     if (fmt == null) {
       fmt = TextOutputFormat.class;
@@ -802,7 +811,9 @@
       c = StreamUtil.goodClassOrNull(partitionerSpec_, defaultPackage);
       if (c != null) {
         jobConf_.setPartitionerClass(c);
-      } 
+      } else {
+        fail("-partitioner : class not found : " + partitionerSpec_);
+      }
     }
     
     if (numReduceTasksSpec_!= null) {

Modified: lucene/hadoop/trunk/src/contrib/streaming/src/test/org/apache/hadoop/streaming/TestGzipInput.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/streaming/src/test/org/apache/hadoop/streaming/TestGzipInput.java?rev=592857&r1=592856&r2=592857&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/contrib/streaming/src/test/org/apache/hadoop/streaming/TestGzipInput.java
(original)
+++ lucene/hadoop/trunk/src/contrib/streaming/src/test/org/apache/hadoop/streaming/TestGzipInput.java
Wed Nov  7 11:12:40 2007
@@ -47,7 +47,6 @@
       "-input", INPUT_FILE.getAbsolutePath(),
       "-output", OUTPUT_DIR.getAbsolutePath(),
       "-mapper", map,
-      "-combiner", combine,
       "-reducer", reduce,
       "-jobconf", "stream.recordreader.compression=gzip"
     };

Modified: lucene/hadoop/trunk/src/contrib/streaming/src/test/org/apache/hadoop/streaming/TestStreaming.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/streaming/src/test/org/apache/hadoop/streaming/TestStreaming.java?rev=592857&r1=592856&r2=592857&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/contrib/streaming/src/test/org/apache/hadoop/streaming/TestStreaming.java
(original)
+++ lucene/hadoop/trunk/src/contrib/streaming/src/test/org/apache/hadoop/streaming/TestStreaming.java
Wed Nov  7 11:12:40 2007
@@ -40,7 +40,6 @@
   protected String map = StreamUtil.makeJavaCommand(TrApp.class, new String[]{".", "\\n"});
   // reduce behave like /usr/bin/uniq. But also prepend lines with R.
   // command-line combiner does not have any effect any more.
-  protected String combine  = StreamUtil.makeJavaCommand(UniqApp.class, new String[]{"C"});
   protected String reduce = StreamUtil.makeJavaCommand(UniqApp.class, new String[]{"R"});
   protected String outputExpect = "Rare\t\nRblue\t\nRbunnies\t\nRpink\t\nRred\t\nRroses\t\nRviolets\t\n";
 
@@ -66,7 +65,6 @@
       "-input", INPUT_FILE.getAbsolutePath(),
       "-output", OUTPUT_DIR.getAbsolutePath(),
       "-mapper", map,
-      "-combiner", combine,
       "-reducer", reduce,
       //"-verbose",
       //"-jobconf", "stream.debug=set"

Modified: lucene/hadoop/trunk/src/contrib/streaming/src/test/org/apache/hadoop/streaming/TestStreamingFailure.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/streaming/src/test/org/apache/hadoop/streaming/TestStreamingFailure.java?rev=592857&r1=592856&r2=592857&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/contrib/streaming/src/test/org/apache/hadoop/streaming/TestStreamingFailure.java
(original)
+++ lucene/hadoop/trunk/src/contrib/streaming/src/test/org/apache/hadoop/streaming/TestStreamingFailure.java
Wed Nov  7 11:12:40 2007
@@ -47,7 +47,6 @@
       "-input", INVALID_INPUT_FILE.getAbsolutePath(),
       "-output", OUTPUT_DIR.getAbsolutePath(),
       "-mapper", map,
-      "-combiner", combine,
       "-reducer", reduce,
       //"-verbose",
       //"-jobconf", "stream.debug=set"



Mime
View raw message