hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cnaur...@apache.org
Subject [2/2] hadoop git commit: HADOOP-13138. Unable to append to a SequenceFile with Compression.NONE. Contributed by Vinayakumar B.
Date Wed, 18 May 2016 16:39:34 GMT
HADOOP-13138. Unable to append to a SequenceFile with Compression.NONE. Contributed by Vinayakumar
B.

(cherry picked from commit 0a527196dcb35a699e12121165bc3738766b569e)
(cherry picked from commit 42f64b3bcc333fa416a8311b84c5b8112043eefd)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4ce5a58c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4ce5a58c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4ce5a58c

Branch: refs/heads/branch-2.8
Commit: 4ce5a58ca964c54bd9a5a570ede3fef39aa6ac6e
Parents: 01a0730
Author: Chris Nauroth <cnauroth@apache.org>
Authored: Wed May 18 09:32:00 2016 -0700
Committer: Chris Nauroth <cnauroth@apache.org>
Committed: Wed May 18 09:32:24 2016 -0700

----------------------------------------------------------------------
 .../java/org/apache/hadoop/io/SequenceFile.java |  7 +-
 .../hadoop/io/TestSequenceFileAppend.java       | 67 ++++++++++++++++++++
 2 files changed, 72 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/4ce5a58c/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SequenceFile.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SequenceFile.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SequenceFile.java
index 149152f..b2eaaff 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SequenceFile.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SequenceFile.java
@@ -1115,9 +1115,12 @@ public class SequenceFile {
             CompressionOption readerCompressionOption = new CompressionOption(
                 reader.getCompressionType(), reader.getCompressionCodec());
 
+            // Codec comparison will be ignored if the compression is NONE
             if (readerCompressionOption.value != compressionTypeOption.value
-                || !readerCompressionOption.codec.getClass().getName()
-                    .equals(compressionTypeOption.codec.getClass().getName())) {
+                || (readerCompressionOption.value != CompressionType.NONE
+                    && readerCompressionOption.codec
+                        .getClass() != compressionTypeOption.codec
+                            .getClass())) {
               throw new IllegalArgumentException(
                   "Compression option provided does not match the file");
             }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4ce5a58c/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSequenceFileAppend.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSequenceFileAppend.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSequenceFileAppend.java
index be4ab92..60d266e 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSequenceFileAppend.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSequenceFileAppend.java
@@ -249,6 +249,73 @@ public class TestSequenceFileAppend {
   }
 
   @Test(timeout = 30000)
+  public void testAppendNoneCompression() throws Exception {
+    Path file = new Path(ROOT_PATH, "testseqappendnonecompr.seq");
+    fs.delete(file, true);
+
+    Option compressOption = Writer.compression(CompressionType.NONE);
+    Writer writer =
+        SequenceFile.createWriter(conf, SequenceFile.Writer.file(file),
+            SequenceFile.Writer.keyClass(Long.class),
+            SequenceFile.Writer.valueClass(String.class), compressOption);
+
+    writer.append(1L, "one");
+    writer.append(2L, "two");
+    writer.close();
+
+    verify2Values(file);
+
+    writer = SequenceFile.createWriter(conf, SequenceFile.Writer.file(file),
+        SequenceFile.Writer.keyClass(Long.class),
+        SequenceFile.Writer.valueClass(String.class),
+        SequenceFile.Writer.appendIfExists(true), compressOption);
+
+    writer.append(3L, "three");
+    writer.append(4L, "four");
+    writer.close();
+
+    verifyAll4Values(file);
+
+    // Verify failure if the compression details are different or not Provided
+    try {
+      writer = SequenceFile.createWriter(conf, SequenceFile.Writer.file(file),
+          SequenceFile.Writer.keyClass(Long.class),
+          SequenceFile.Writer.valueClass(String.class),
+          SequenceFile.Writer.appendIfExists(true));
+      writer.close();
+      fail("Expected IllegalArgumentException for compression options");
+    } catch (IllegalArgumentException iae) {
+      // Expected exception. Ignore it
+    }
+
+    // Verify failure if the compression details are different
+    try {
+      Option wrongCompressOption =
+          Writer.compression(CompressionType.RECORD, new GzipCodec());
+
+      writer = SequenceFile.createWriter(conf, SequenceFile.Writer.file(file),
+          SequenceFile.Writer.keyClass(Long.class),
+          SequenceFile.Writer.valueClass(String.class),
+          SequenceFile.Writer.appendIfExists(true), wrongCompressOption);
+      writer.close();
+      fail("Expected IllegalArgumentException for compression options");
+    } catch (IllegalArgumentException iae) {
+      // Expected exception. Ignore it
+    }
+
+    // Codec should be ignored
+    Option noneWithCodec =
+        Writer.compression(CompressionType.NONE, new DefaultCodec());
+
+    writer = SequenceFile.createWriter(conf, SequenceFile.Writer.file(file),
+        SequenceFile.Writer.keyClass(Long.class),
+        SequenceFile.Writer.valueClass(String.class),
+        SequenceFile.Writer.appendIfExists(true), noneWithCodec);
+    writer.close();
+    fs.deleteOnExit(file);
+  }
+
+  @Test(timeout = 30000)
   public void testAppendSort() throws Exception {
     GenericTestUtils.assumeInNativeProfile();
 


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message