Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id D6FC02009D9 for ; Thu, 19 May 2016 21:47:05 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id D5B2D160A1D; Thu, 19 May 2016 19:47:05 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 0B2EF1609AE for ; Thu, 19 May 2016 21:47:04 +0200 (CEST) Received: (qmail 50349 invoked by uid 500); 19 May 2016 19:46:51 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 49005 invoked by uid 99); 19 May 2016 19:46:51 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 19 May 2016 19:46:51 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id EE7D3DFF13; Thu, 19 May 2016 19:46:50 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: arp@apache.org To: common-commits@hadoop.apache.org Date: Thu, 19 May 2016 19:47:27 -0000 Message-Id: <99208a9faca64fe7ab2dbd0a8b0a4d57@git.apache.org> In-Reply-To: <25255808f6d94e1e9958e0ef532b7599@git.apache.org> References: <25255808f6d94e1e9958e0ef532b7599@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [38/50] [abbrv] hadoop git commit: HADOOP-13138. Unable to append to a SequenceFile with Compression.NONE. Contributed by Vinayakumar B. archived-at: Thu, 19 May 2016 19:47:06 -0000 HADOOP-13138. Unable to append to a SequenceFile with Compression.NONE. Contributed by Vinayakumar B. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bad85f3e Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bad85f3e Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bad85f3e Branch: refs/heads/HDFS-7240 Commit: bad85f3e3a53b5fb692c77df6c11dcf968ad7e62 Parents: cf552aa Author: Chris Nauroth Authored: Wed May 18 09:32:00 2016 -0700 Committer: Chris Nauroth Committed: Wed May 18 09:39:43 2016 -0700 ---------------------------------------------------------------------- .../java/org/apache/hadoop/io/SequenceFile.java | 7 +- .../hadoop/io/TestSequenceFileAppend.java | 67 ++++++++++++++++++++ 2 files changed, 72 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/bad85f3e/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SequenceFile.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SequenceFile.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SequenceFile.java index ed57eee..0f3f317 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SequenceFile.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SequenceFile.java @@ -1115,9 +1115,12 @@ public class SequenceFile { CompressionOption readerCompressionOption = new CompressionOption( reader.getCompressionType(), reader.getCompressionCodec()); + // Codec comparison will be ignored if the compression is NONE if (readerCompressionOption.value != compressionTypeOption.value - || !readerCompressionOption.codec.getClass().getName() - .equals(compressionTypeOption.codec.getClass().getName())) { + || (readerCompressionOption.value != CompressionType.NONE + && readerCompressionOption.codec + .getClass() != compressionTypeOption.codec + .getClass())) { throw new IllegalArgumentException( "Compression option provided does not match the file"); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/bad85f3e/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSequenceFileAppend.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSequenceFileAppend.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSequenceFileAppend.java index 9e49c9e..b31c809 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSequenceFileAppend.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSequenceFileAppend.java @@ -249,6 +249,73 @@ public class TestSequenceFileAppend { } @Test(timeout = 30000) + public void testAppendNoneCompression() throws Exception { + Path file = new Path(ROOT_PATH, "testseqappendnonecompr.seq"); + fs.delete(file, true); + + Option compressOption = Writer.compression(CompressionType.NONE); + Writer writer = + SequenceFile.createWriter(conf, SequenceFile.Writer.file(file), + SequenceFile.Writer.keyClass(Long.class), + SequenceFile.Writer.valueClass(String.class), compressOption); + + writer.append(1L, "one"); + writer.append(2L, "two"); + writer.close(); + + verify2Values(file); + + writer = SequenceFile.createWriter(conf, SequenceFile.Writer.file(file), + SequenceFile.Writer.keyClass(Long.class), + SequenceFile.Writer.valueClass(String.class), + SequenceFile.Writer.appendIfExists(true), compressOption); + + writer.append(3L, "three"); + writer.append(4L, "four"); + writer.close(); + + verifyAll4Values(file); + + // Verify failure if the compression details are different or not Provided + try { + writer = SequenceFile.createWriter(conf, SequenceFile.Writer.file(file), + SequenceFile.Writer.keyClass(Long.class), + SequenceFile.Writer.valueClass(String.class), + SequenceFile.Writer.appendIfExists(true)); + writer.close(); + fail("Expected IllegalArgumentException for compression options"); + } catch (IllegalArgumentException iae) { + // Expected exception. Ignore it + } + + // Verify failure if the compression details are different + try { + Option wrongCompressOption = + Writer.compression(CompressionType.RECORD, new GzipCodec()); + + writer = SequenceFile.createWriter(conf, SequenceFile.Writer.file(file), + SequenceFile.Writer.keyClass(Long.class), + SequenceFile.Writer.valueClass(String.class), + SequenceFile.Writer.appendIfExists(true), wrongCompressOption); + writer.close(); + fail("Expected IllegalArgumentException for compression options"); + } catch (IllegalArgumentException iae) { + // Expected exception. Ignore it + } + + // Codec should be ignored + Option noneWithCodec = + Writer.compression(CompressionType.NONE, new DefaultCodec()); + + writer = SequenceFile.createWriter(conf, SequenceFile.Writer.file(file), + SequenceFile.Writer.keyClass(Long.class), + SequenceFile.Writer.valueClass(String.class), + SequenceFile.Writer.appendIfExists(true), noneWithCodec); + writer.close(); + fs.deleteOnExit(file); + } + + @Test(timeout = 30000) public void testAppendSort() throws Exception { GenericTestUtils.assumeInNativeProfile(); --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org