Mailing-List: contact mapreduce-commits-help@hadoop.apache.org; run by ezmlm
Precedence: bulk
Reply-To: mapreduce-dev@hadoop.apache.org
Content-Type: text/plain; charset="utf-8"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
Subject: svn commit: r1035357 - in /hadoop/mapreduce/trunk/src:
 java/org/apache/hadoop/mapred/IFile.java
 test/mapred/org/apache/hadoop/mapred/TestIFile.java
Date: Mon, 15 Nov 2010 17:16:25 -0000
To: mapreduce-commits@hadoop.apache.org
From: tomwhite@apache.org
Message-Id: <20101115171625.B5CF0238890B@eris.apache.org>

Author: tomwhite
Date: Mon Nov 15 17:16:25 2010
New Revision: 1035357

URL: http://svn.apache.org/viewvc?rev=1035357&view=rev
Log:
MAPREDUCE-1784. IFile should check for null compressor. Contributed by Eli Collins.

Added:
    hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestIFile.java   (with props)
Modified:
    hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/IFile.java

Modified: hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/IFile.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/IFile.java?rev=1035357&r1=1035356&r2=1035357&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/IFile.java (original)
+++ hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/IFile.java Mon Nov 15 17:16:25 2010
@@ -42,6 +42,9 @@ import org.apache.hadoop.io.compress.Dec
 import org.apache.hadoop.io.serializer.SerializationFactory;
 import org.apache.hadoop.io.serializer.Serializer;
 
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
 /**
  * <code>IFile</code> is the simple <key-len, value-len, key, value> format
  * for the intermediate map-outputs in Map-Reduce.
@@ -52,7 +55,7 @@ import org.apache.hadoop.io.serializer.S
 @InterfaceAudience.Private
 @InterfaceStability.Unstable
 public class IFile {
-
+  private static final Log LOG = LogFactory.getLog(IFile.class);
   public static final int EOF_MARKER = -1; // End of File Marker
   
   /**
@@ -107,13 +110,17 @@ public class IFile {
       this.checksumOut = new IFileOutputStream(out);
       this.rawOut = out;
       this.start = this.rawOut.getPos();
-      
       if (codec != null) {
         this.compressor = CodecPool.getCompressor(codec);
-        this.compressor.reset();
-        this.compressedOut = codec.createOutputStream(checksumOut, compressor);
-        this.out = new FSDataOutputStream(this.compressedOut,  null);
-        this.compressOutput = true;
+        if (this.compressor != null) {
+          this.compressor.reset();
+          this.compressedOut = codec.createOutputStream(checksumOut, compressor);
+          this.out = new FSDataOutputStream(this.compressedOut,  null);
+          this.compressOutput = true;
+        } else {
+          LOG.warn("Could not obtain compressor from CodecPool");
+          this.out = new FSDataOutputStream(checksumOut,null);
+        }
       } else {
         this.out = new FSDataOutputStream(checksumOut,null);
       }
@@ -335,7 +342,12 @@ public class IFile {
       checksumIn = new IFileInputStream(in,length);
       if (codec != null) {
         decompressor = CodecPool.getDecompressor(codec);
-        this.in = codec.createInputStream(checksumIn, decompressor);
+        if (decompressor != null) {
+          this.in = codec.createInputStream(checksumIn, decompressor);
+        } else {
+          LOG.warn("Could not obtain decompressor from CodecPool");
+          this.in = checksumIn;
+        }
       } else {
         this.in = checksumIn;
       }

Added: hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestIFile.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestIFile.java?rev=1035357&view=auto
==============================================================================
--- hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestIFile.java (added)
+++ hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestIFile.java Mon Nov 15 17:16:25 2010
@@ -0,0 +1,63 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocalFileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.compress.DefaultCodec;
+import org.apache.hadoop.io.compress.GzipCodec;
+
+import org.junit.Test;
+
+public class TestIFile {
+
+  @Test
+  /**
+   * Create an IFile.Writer using GzipCodec since this codec does not
+   * have a compressor when run via the tests (ie no native libraries).
+   */
+  public void testIFileWriterWithCodec() throws Exception {
+    Configuration conf = new Configuration();
+    FileSystem localFs = FileSystem.getLocal(conf);
+    FileSystem rfs = ((LocalFileSystem)localFs).getRaw();
+    Path path = new Path(new Path("build/test.ifile"), "data");
+    DefaultCodec codec = new GzipCodec();
+    codec.setConf(conf);
+    IFile.Writer<Text, Text> writer =
+      new IFile.Writer<Text, Text>(conf, rfs, path, Text.class, Text.class,
+                                   codec, null);
+    writer.close();
+  }
+
+  @Test
+  /** Same as above but create a reader. */
+  public void testIFileReaderWithCodec() throws Exception {
+    Configuration conf = new Configuration();
+    FileSystem localFs = FileSystem.getLocal(conf);
+    FileSystem rfs = ((LocalFileSystem)localFs).getRaw();
+    Path path = new Path(new Path("build/test.ifile"), "data");
+    DefaultCodec codec = new GzipCodec();
+    codec.setConf(conf);
+    IFile.Reader<Text, Text> reader =
+      new IFile.Reader<Text, Text>(conf, rfs, path, codec, null);
+    reader.close();
+  }
+}

Propchange: hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestIFile.java
------------------------------------------------------------------------------
    svn:eol-style = native