Return-Path: X-Original-To: apmail-commons-commits-archive@minotaur.apache.org Delivered-To: apmail-commons-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id BD906D934 for ; Fri, 10 Aug 2012 13:52:34 +0000 (UTC) Received: (qmail 62543 invoked by uid 500); 10 Aug 2012 13:52:34 -0000 Delivered-To: apmail-commons-commits-archive@commons.apache.org Received: (qmail 62441 invoked by uid 500); 10 Aug 2012 13:52:34 -0000 Mailing-List: contact commits-help@commons.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@commons.apache.org Delivered-To: mailing list commits@commons.apache.org Received: (qmail 62434 invoked by uid 99); 10 Aug 2012 13:52:34 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 10 Aug 2012 13:52:34 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 10 Aug 2012 13:52:32 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 2FD392388860 for ; Fri, 10 Aug 2012 13:51:49 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1371703 - /commons/proper/io/trunk/src/test/java/org/apache/commons/io/input/BOMInputStreamTest.java Date: Fri, 10 Aug 2012 13:51:49 -0000 To: commits@commons.apache.org From: ggregory@apache.org X-Mailer: svnmailer-1.0.8-patched Message-Id: <20120810135149.2FD392388860@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: ggregory Date: Fri Aug 10 13:51:48 2012 New Revision: 1371703 URL: http://svn.apache.org/viewvc?rev=1371703&view=rev Log: More tests while working out possible issue with UTF-32. Modified: commons/proper/io/trunk/src/test/java/org/apache/commons/io/input/BOMInputStreamTest.java Modified: commons/proper/io/trunk/src/test/java/org/apache/commons/io/input/BOMInputStreamTest.java URL: http://svn.apache.org/viewvc/commons/proper/io/trunk/src/test/java/org/apache/commons/io/input/BOMInputStreamTest.java?rev=1371703&r1=1371702&r2=1371703&view=diff ============================================================================== --- commons/proper/io/trunk/src/test/java/org/apache/commons/io/input/BOMInputStreamTest.java (original) +++ commons/proper/io/trunk/src/test/java/org/apache/commons/io/input/BOMInputStreamTest.java Fri Aug 10 13:51:48 2012 @@ -23,6 +23,7 @@ import java.io.InputStream; import junit.framework.TestCase; import org.apache.commons.io.ByteOrderMark; +import org.apache.commons.io.Charsets; import org.junit.Assert; /** @@ -38,7 +39,7 @@ public class BOMInputStreamTest extends /** * Creates the underlying data stream, with or without BOM. */ - public InputStream createDataStream(byte[] baseData, boolean addBOM) { + public InputStream createUtf8DataStream(byte[] baseData, boolean addBOM) { byte[] data = baseData; if (addBOM) { data = new byte[baseData.length + 3]; @@ -50,6 +51,34 @@ public class BOMInputStreamTest extends return new ByteArrayInputStream(data); } + /** + * Creates the underlying data stream, with or without BOM. + */ + public InputStream createUtf16BeDataStream(byte[] baseData, boolean addBOM) { + byte[] data = baseData; + if (addBOM) { + data = new byte[baseData.length + 2]; + data[0] = (byte) 0xFE; + data[1] = (byte) 0xFF; + System.arraycopy(baseData, 0, data, 2, baseData.length); + } + return new ByteArrayInputStream(data); + } + + /** + * Creates the underlying data stream, with or without BOM. + */ + public InputStream createUtf16LeDataStream(byte[] baseData, boolean addBOM) { + byte[] data = baseData; + if (addBOM) { + data = new byte[baseData.length + 2]; + data[0] = (byte) 0xFF; + data[1] = (byte) 0xFE; + System.arraycopy(baseData, 0, data, 2, baseData.length); + } + return new ByteArrayInputStream(data); + } + private void assertData(byte[] expected, byte[] actual, int len) throws Exception { assertEquals("length", expected.length, len); @@ -85,7 +114,7 @@ public class BOMInputStreamTest extends // make sure that our support code works as expected public void testSupportCode() throws Exception { - InputStream in = createDataStream(new byte[] { 'A', 'B' }, true); + InputStream in = createUtf8DataStream(new byte[] { 'A', 'B' }, true); byte[] buf = new byte[1024]; int len = in.read(buf); assertEquals(5, len); @@ -102,7 +131,7 @@ public class BOMInputStreamTest extends public void testReadWithoutBOM() throws Exception { byte[] data = new byte[] { 'A', 'B', 'C' }; - BOMInputStream in = new BOMInputStream(createDataStream(data, false)); + BOMInputStream in = new BOMInputStream(createUtf8DataStream(data, false)); assertEquals('A', in.read()); assertEquals('B', in.read()); assertEquals('C', in.read()); @@ -114,7 +143,7 @@ public class BOMInputStreamTest extends public void testReadEmpty() throws Exception { byte[] data = new byte[] {}; - BOMInputStream in = new BOMInputStream(createDataStream(data, false)); + BOMInputStream in = new BOMInputStream(createUtf8DataStream(data, false)); assertEquals(-1, in.read()); assertFalse("hasBOM()", in.hasBOM()); assertFalse("hasBOM(UTF-8)", in.hasBOM(ByteOrderMark.UTF_8)); @@ -123,7 +152,7 @@ public class BOMInputStreamTest extends public void testReadSmall() throws Exception { byte[] data = new byte[] { 'A', 'B' }; - BOMInputStream in = new BOMInputStream(createDataStream(data, false)); + BOMInputStream in = new BOMInputStream(createUtf8DataStream(data, false)); assertEquals('A', in.read()); assertEquals('B', in.read()); assertEquals(-1, in.read()); @@ -132,9 +161,27 @@ public class BOMInputStreamTest extends assertNull("getBOM", in.getBOM()); } - public void testReadWithBOM() throws Exception { + public void testReadWithBOMUtf8Bytes() throws Exception { byte[] data = new byte[] { 'A', 'B', 'C' }; - BOMInputStream in = new BOMInputStream(createDataStream(data, true)); + BOMInputStream in = new BOMInputStream(createUtf8DataStream(data, true)); + assertEquals('A', in.read()); + assertEquals('B', in.read()); + assertEquals('C', in.read()); + assertEquals(-1, in.read()); + assertTrue("hasBOM()", in.hasBOM()); + assertTrue("hasBOM(UTF-8)", in.hasBOM(ByteOrderMark.UTF_8)); + assertEquals("getBOM", ByteOrderMark.UTF_8, in.getBOM()); + try { + in.hasBOM(ByteOrderMark.UTF_16BE); + fail("Expected IllegalArgumentException"); + } catch (IllegalArgumentException e) { + // expected - not configured for UTF-16BE + } + } + + public void testReadWithBOMUtf8String() throws Exception { + byte[] data = "ABC".getBytes(Charsets.UTF_8); + BOMInputStream in = new BOMInputStream(createUtf8DataStream(data, true)); assertEquals('A', in.read()); assertEquals('B', in.read()); assertEquals('C', in.read()); @@ -144,6 +191,49 @@ public class BOMInputStreamTest extends assertEquals("getBOM", ByteOrderMark.UTF_8, in.getBOM()); try { in.hasBOM(ByteOrderMark.UTF_16BE); + fail("Expected IllegalArgumentException"); + } catch (IllegalArgumentException e) { + // expected - not configured for UTF-16BE + } + } + + public void testReadWithBOMUtf16Be() throws Exception { + byte[] data = "ABC".getBytes(Charsets.UTF_16BE); + BOMInputStream in = new BOMInputStream(createUtf16BeDataStream(data, true), ByteOrderMark.UTF_16BE); + assertEquals(0, in.read()); + assertEquals('A', in.read()); + assertEquals(0, in.read()); + assertEquals('B', in.read()); + assertEquals(0, in.read()); + assertEquals('C', in.read()); + assertEquals(-1, in.read()); + assertTrue("hasBOM()", in.hasBOM()); + assertTrue("hasBOM(UTF-16BE)", in.hasBOM(ByteOrderMark.UTF_16BE)); + assertEquals("getBOM", ByteOrderMark.UTF_16BE, in.getBOM()); + try { + in.hasBOM(ByteOrderMark.UTF_16LE); + fail("Expected IllegalArgumentException"); + } catch (IllegalArgumentException e) { + // expected - not configured for UTF-16LE + } + } + + public void testReadWithBOMUtf16Le() throws Exception { + byte[] data = "ABC".getBytes(Charsets.UTF_16LE); + BOMInputStream in = new BOMInputStream(createUtf16LeDataStream(data, true), ByteOrderMark.UTF_16LE); + assertEquals('A', in.read()); + assertEquals(0, in.read()); + assertEquals('B', in.read()); + assertEquals(0, in.read()); + assertEquals('C', in.read()); + assertEquals(0, in.read()); + assertEquals(-1, in.read()); + assertTrue("hasBOM()", in.hasBOM()); + assertTrue("hasBOM(UTF-16LE)", in.hasBOM(ByteOrderMark.UTF_16LE)); + assertEquals("getBOM", ByteOrderMark.UTF_16LE, in.getBOM()); + try { + in.hasBOM(ByteOrderMark.UTF_16BE); + fail("Expected IllegalArgumentException"); } catch (IllegalArgumentException e) { // expected - not configured for UTF-16BE } @@ -151,7 +241,7 @@ public class BOMInputStreamTest extends public void testGetBOMFirstThenRead() throws Exception { byte[] data = new byte[] { 'A', 'B', 'C' }; - BOMInputStream in = new BOMInputStream(createDataStream(data, true)); + BOMInputStream in = new BOMInputStream(createUtf8DataStream(data, true)); assertEquals("getBOM", ByteOrderMark.UTF_8, in.getBOM()); assertTrue("hasBOM()", in.hasBOM()); assertTrue("hasBOM(UTF-8)", in.hasBOM(ByteOrderMark.UTF_8)); @@ -163,7 +253,7 @@ public class BOMInputStreamTest extends public void testReadWithBOMInclude() throws Exception { byte[] data = new byte[] { 'A', 'B', 'C' }; - BOMInputStream in = new BOMInputStream(createDataStream(data, true), true); + BOMInputStream in = new BOMInputStream(createUtf8DataStream(data, true), true); assertEquals(0xEF, in.read()); assertEquals(0xBB, in.read()); assertEquals(0xBF, in.read()); @@ -178,7 +268,7 @@ public class BOMInputStreamTest extends public void testGetBOMFirstThenReadInclude() throws Exception { byte[] data = new byte[] { 'A', 'B', 'C' }; - BOMInputStream in = new BOMInputStream(createDataStream(data, true), true); + BOMInputStream in = new BOMInputStream(createUtf8DataStream(data, true), true); assertTrue("hasBOM()", in.hasBOM()); assertTrue("hasBOM(UTF-8)", in.hasBOM(ByteOrderMark.UTF_8)); assertEquals("getBOM", ByteOrderMark.UTF_8, in.getBOM()); @@ -193,7 +283,7 @@ public class BOMInputStreamTest extends public void testReadWithMultipleBOM() throws Exception { byte[] data = new byte[] { 'A', 'B', 'C' }; - BOMInputStream in = new BOMInputStream(createDataStream(data, true), + BOMInputStream in = new BOMInputStream(createUtf8DataStream(data, true), ByteOrderMark.UTF_16BE, ByteOrderMark.UTF_8); assertEquals('A', in.read()); assertEquals('B', in.read()); @@ -207,35 +297,35 @@ public class BOMInputStreamTest extends public void testEmptyBufferWithoutBOM() throws Exception { byte[] data = new byte[] {}; - InputStream in = new BOMInputStream(createDataStream(data, false)); + InputStream in = new BOMInputStream(createUtf8DataStream(data, false)); byte[] buf = new byte[1024]; assertEquals(-1, in.read(buf)); } public void testEmptyBufferWithBOM() throws Exception { byte[] data = new byte[] {}; - InputStream in = new BOMInputStream(createDataStream(data, true)); + InputStream in = new BOMInputStream(createUtf8DataStream(data, true)); byte[] buf = new byte[1024]; assertEquals(-1, in.read(buf)); } public void testLargeBufferWithoutBOM() throws Exception { byte[] data = new byte[] { 'A', 'B', 'C' }; - InputStream in = new BOMInputStream(createDataStream(data, false)); + InputStream in = new BOMInputStream(createUtf8DataStream(data, false)); byte[] buf = new byte[1024]; assertData(data, buf, in.read(buf)); } public void testLargeBufferWithBOM() throws Exception { byte[] data = new byte[] { 'A', 'B', 'C' }; - InputStream in = new BOMInputStream(createDataStream(data, true)); + InputStream in = new BOMInputStream(createUtf8DataStream(data, true)); byte[] buf = new byte[1024]; assertData(data, buf, in.read(buf)); } public void testSmallBufferWithoutBOM() throws Exception { byte[] data = new byte[] { 'A', 'B', 'C' }; - InputStream in = new BOMInputStream(createDataStream(data, false)); + InputStream in = new BOMInputStream(createUtf8DataStream(data, false)); byte[] buf = new byte[1024]; assertData(new byte[] { 'A', 'B' }, buf, in.read(buf, 0, 2)); assertData(new byte[] { 'C' }, buf, in.read(buf, 0, 2)); @@ -243,7 +333,7 @@ public class BOMInputStreamTest extends public void testSmallBufferWithBOM() throws Exception { byte[] data = new byte[] { 'A', 'B', 'C' }; - InputStream in = new BOMInputStream(createDataStream(data, true)); + InputStream in = new BOMInputStream(createUtf8DataStream(data, true)); byte[] buf = new byte[1024]; assertData(new byte[] { 'A', 'B' }, buf, in.read(buf, 0, 2)); assertData(new byte[] { 'C' }, buf, in.read(buf, 0, 2)); @@ -251,7 +341,7 @@ public class BOMInputStreamTest extends public void testLeadingNonBOMSingleRead() throws Exception { byte[] data = new byte[] { (byte) 0xEF, (byte) 0xAB, (byte) 0xCD }; - InputStream in = new BOMInputStream(createDataStream(data, false)); + InputStream in = new BOMInputStream(createUtf8DataStream(data, false)); assertEquals(0xEF, in.read()); assertEquals(0xAB, in.read()); assertEquals(0xCD, in.read()); @@ -260,28 +350,28 @@ public class BOMInputStreamTest extends public void testLeadingNonBOMBufferedRead() throws Exception { byte[] data = new byte[] { (byte) 0xEF, (byte) 0xAB, (byte) 0xCD }; - InputStream in = new BOMInputStream(createDataStream(data, false)); + InputStream in = new BOMInputStream(createUtf8DataStream(data, false)); byte[] buf = new byte[1024]; assertData(data, buf, in.read(buf)); } public void testSkipWithoutBOM() throws Exception { byte[] data = new byte[] { 'A', 'B', 'C', 'D' }; - InputStream in = new BOMInputStream(createDataStream(data, false)); + InputStream in = new BOMInputStream(createUtf8DataStream(data, false)); in.skip(2L); assertEquals('C', in.read()); } public void testSkipWithBOM() throws Exception { byte[] data = new byte[] { 'A', 'B', 'C', 'D' }; - InputStream in = new BOMInputStream(createDataStream(data, true)); + InputStream in = new BOMInputStream(createUtf8DataStream(data, true)); in.skip(2L); assertEquals('C', in.read()); } public void testMarkResetAfterReadWithoutBOM() throws Exception { byte[] data = new byte[] { 'A', 'B', 'C', 'D' }; - InputStream in = new BOMInputStream(createDataStream(data, false)); + InputStream in = new BOMInputStream(createUtf8DataStream(data, false)); assertTrue(in.markSupported()); in.read(); @@ -295,7 +385,7 @@ public class BOMInputStreamTest extends public void testMarkResetAfterReadWithBOM() throws Exception { byte[] data = new byte[] { 'A', 'B', 'C', 'D' }; - InputStream in = new BOMInputStream(createDataStream(data, true)); + InputStream in = new BOMInputStream(createUtf8DataStream(data, true)); assertTrue(in.markSupported()); in.read(); @@ -309,7 +399,7 @@ public class BOMInputStreamTest extends public void testMarkResetBeforeReadWithoutBOM() throws Exception { byte[] data = new byte[] { 'A', 'B', 'C', 'D' }; - InputStream in = new BOMInputStream(createDataStream(data, false)); + InputStream in = new BOMInputStream(createUtf8DataStream(data, false)); assertTrue(in.markSupported()); in.mark(10); @@ -322,7 +412,7 @@ public class BOMInputStreamTest extends public void testMarkResetBeforeReadWithBOM() throws Exception { byte[] data = new byte[] { 'A', 'B', 'C', 'D' }; - InputStream in = new BOMInputStream(createDataStream(data, true)); + InputStream in = new BOMInputStream(createUtf8DataStream(data, true)); assertTrue(in.markSupported()); in.mark(10); @@ -335,26 +425,26 @@ public class BOMInputStreamTest extends public void testAvailableWithoutBOM() throws Exception { byte[] data = new byte[] { 'A', 'B', 'C', 'D' }; - InputStream in = new BOMInputStream(createDataStream(data, false)); + InputStream in = new BOMInputStream(createUtf8DataStream(data, false)); assertEquals(4, in.available()); } public void testAvailableWithBOM() throws Exception { byte[] data = new byte[] { 'A', 'B', 'C', 'D' }; - InputStream in = new BOMInputStream(createDataStream(data, true)); + InputStream in = new BOMInputStream(createUtf8DataStream(data, true)); assertEquals(7, in.available()); } public void testNoBoms() throws Exception { byte[] data = new byte[] { 'A', 'B', 'C' }; try { - new BOMInputStream(createDataStream(data, true), false, (ByteOrderMark[])null); + new BOMInputStream(createUtf8DataStream(data, true), false, (ByteOrderMark[])null); fail("Null BOMs, expected IllegalArgumentException"); } catch (IllegalArgumentException e) { // expected } try { - new BOMInputStream(createDataStream(data, true), false, new ByteOrderMark[0]); + new BOMInputStream(createUtf8DataStream(data, true), false, new ByteOrderMark[0]); fail("Null BOMs, expected IllegalArgumentException"); } catch (IllegalArgumentException e) { // expected