lucenenet-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From nightowl...@apache.org
Subject [50/50] [abbrv] lucenenet git commit: Ported tests for Support.DataInputStream and Support.DataOutputStream from Java and fixed bugs.
Date Sun, 23 Oct 2016 13:02:36 GMT
Ported tests for Support.DataInputStream and Support.DataOutputStream from Java and fixed bugs.


Project: http://git-wip-us.apache.org/repos/asf/lucenenet/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucenenet/commit/dec66102
Tree: http://git-wip-us.apache.org/repos/asf/lucenenet/tree/dec66102
Diff: http://git-wip-us.apache.org/repos/asf/lucenenet/diff/dec66102

Branch: refs/heads/master
Commit: dec66102d31b091a03a3b8b4233d87f1b03e86ad
Parents: 2952508
Author: Shad Storhaug <shad@shadstorhaug.com>
Authored: Thu Oct 6 00:26:59 2016 +0700
Committer: Shad Storhaug <shad@shadstorhaug.com>
Committed: Sun Oct 23 19:44:35 2016 +0700

----------------------------------------------------------------------
 src/Lucene.Net.Core/Support/DataInputStream.cs  |  20 +-
 src/Lucene.Net.Core/Support/DataOutputStream.cs |   9 +
 src/Lucene.Net.Tests/Lucene.Net.Tests.csproj    |   5 +
 src/Lucene.Net.Tests/core/Support/ReadFully.txt |  14 ++
 .../core/Support/TestDataInputStream.cs         | 224 +++++++++++++++++++
 .../core/Support/TestDataOutputStream.cs        | 172 ++++++++++++++
 6 files changed, 443 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucenenet/blob/dec66102/src/Lucene.Net.Core/Support/DataInputStream.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Core/Support/DataInputStream.cs b/src/Lucene.Net.Core/Support/DataInputStream.cs
index 30dc6a6..b135125 100644
--- a/src/Lucene.Net.Core/Support/DataInputStream.cs
+++ b/src/Lucene.Net.Core/Support/DataInputStream.cs
@@ -7,6 +7,10 @@ namespace Lucene.Net.Support
     /// Java's DataInputStream is similar to .NET's BinaryReader. However, it reads
     /// using a modified UTF-8 format that cannot be read using BinaryReader.
     /// This is a port of DataInputStream that is fully compatible with Java's DataOutputStream.
+    /// <para>
+    /// Usage Note: Always favor BinaryReader over DataInputStream unless you specifically
need
+    /// the modified UTF-8 format and/or the <see cref="ReadUTF(IDataInput)"/> method.
+    /// </para>
     /// </summary>
     public class DataInputStream : IDataInput, IDisposable
     {
@@ -62,7 +66,7 @@ namespace Lucene.Net.Support
             int total = 0;
             int cur = 0;
 
-            while ((total < n) && ((cur = (int)@in.Seek(n - total, SeekOrigin.Begin))
> 0))
+            while ((total < n) && ((cur = Skip(@in, n - total)) > 0))
             {
                 total += cur;
             }
@@ -70,6 +74,20 @@ namespace Lucene.Net.Support
             return total;
         }
 
+        /// <summary>
+        /// Helper method for SkipBytes, since Position and Seek do not work on
+        /// non-seekable streams.
+        /// </summary>
+        private static int Skip(Stream stream, int n)
+        {
+            int total = 0;
+            while (total < n && stream.ReadByte() > -1)
+            {
+                total++;
+            }
+            return total;
+        }
+
         public bool ReadBoolean()
         {
             int ch = @in.ReadByte();

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/dec66102/src/Lucene.Net.Core/Support/DataOutputStream.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Core/Support/DataOutputStream.cs b/src/Lucene.Net.Core/Support/DataOutputStream.cs
index 518dba7..b1e755e 100644
--- a/src/Lucene.Net.Core/Support/DataOutputStream.cs
+++ b/src/Lucene.Net.Core/Support/DataOutputStream.cs
@@ -8,6 +8,10 @@ namespace Lucene.Net.Support
     /// Java's DataOutputStream is similar to .NET's BinaryWriter. However, it writes
     /// in a modified UTF-8 format that cannot be read (or duplicated) using BinaryWriter.
     /// This is a port of DataOutputStream that is fully compatible with Java's DataInputStream.
+    /// <para>
+    /// Usage Note: Always favor BinaryWriter over DataOutputStream unless you specifically
need
+    /// the modified UTF-8 format and/or the <see cref="WriteUTF(IDataOutput)"/> method.
+    /// </para>
     /// </summary>
     public class DataOutputStream : IDataOutput, IDisposable
     {
@@ -238,6 +242,11 @@ namespace Lucene.Net.Support
             return utflen + 2;
         }
 
+        public int Length
+        {
+            get { return written; }
+        }
+
 
         #region From FilterOutputStream
 

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/dec66102/src/Lucene.Net.Tests/Lucene.Net.Tests.csproj
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Lucene.Net.Tests.csproj b/src/Lucene.Net.Tests/Lucene.Net.Tests.csproj
index e750cca..3664713 100644
--- a/src/Lucene.Net.Tests/Lucene.Net.Tests.csproj
+++ b/src/Lucene.Net.Tests/Lucene.Net.Tests.csproj
@@ -477,6 +477,8 @@
     <Compile Include="core\Support\TestByteBuffer.cs" />
     <Compile Include="core\Support\TestLurchTable.cs" />
     <Compile Include="core\Support\TestLurchTableThreading.cs" />
+    <Compile Include="core\Support\TestDataInputStream.cs" />
+    <Compile Include="core\Support\TestDataOutputStream.cs" />
     <Compile Include="core\TestAssertions.cs">
       <SubType>Code</SubType>
     </Compile>
@@ -676,6 +678,9 @@
   <ItemGroup>
     <Service Include="{82A7F48D-3B50-4B1E-B82E-3ADA8210C358}" />
   </ItemGroup>
+  <ItemGroup>
+    <EmbeddedResource Include="core\Support\ReadFully.txt" />
+  </ItemGroup>
   <Import Project="$(MSBuildToolsPath)\Microsoft.CSHARP.Targets" />
   <ProjectExtensions>
     <VisualStudio AllowExistingFolder="true" />

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/dec66102/src/Lucene.Net.Tests/core/Support/ReadFully.txt
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/core/Support/ReadFully.txt b/src/Lucene.Net.Tests/core/Support/ReadFully.txt
new file mode 100644
index 0000000..ad76b7a
--- /dev/null
+++ b/src/Lucene.Net.Tests/core/Support/ReadFully.txt
@@ -0,0 +1,14 @@
+Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt
ut labore et dolore magna 
+aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip
ex ea commodo consequat. 
+Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla
pariatur. Excepteur 
+sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est
laborum.
+
+Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt
ut labore et dolore magna 
+aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip
ex ea commodo consequat. 
+Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla
pariatur. Excepteur 
+sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est
laborum.
+
+Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt
ut labore et dolore magna 
+aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip
ex ea commodo consequat. 
+Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla
pariatur. Excepteur 
+sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est
laborum.

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/dec66102/src/Lucene.Net.Tests/core/Support/TestDataInputStream.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/core/Support/TestDataInputStream.cs b/src/Lucene.Net.Tests/core/Support/TestDataInputStream.cs
new file mode 100644
index 0000000..5c8463e
--- /dev/null
+++ b/src/Lucene.Net.Tests/core/Support/TestDataInputStream.cs
@@ -0,0 +1,224 @@
+using Lucene.Net.Util;
+using NUnit.Framework;
+using System;
+using System.IO;
+using System.Text;
+
+namespace Lucene.Net.Support
+{
+    public class TestDataInputStream : LuceneTestCase
+    {
+        [Test]
+        public void TestReadFully()
+        {
+            const string READFULLY_TEST_FILE = "Lucene.Net.Tests.core.Support.ReadFully.txt";
+            byte[] buffer = new byte[1367];
+
+            Stream @in = GetType().Assembly.GetManifestResourceStream(READFULLY_TEST_FILE);
+            DataInputStream dis;
+            using (dis = new DataInputStream(@in))
+            { 
+                // Read once for real
+                dis.ReadFully(buffer, 0, 1366);
+            }
+
+            // Read past the end of the stream
+            @in = GetType().Assembly.GetManifestResourceStream(READFULLY_TEST_FILE);
+            dis = new DataInputStream(@in);
+            bool caughtException = false;
+            try
+            {
+                dis.ReadFully(buffer, 0, buffer.Length);
+            }
+#pragma warning disable 168
+            catch (EndOfStreamException ie)
+#pragma warning restore 168
+            {
+                caughtException = true;
+            }
+            finally
+            {
+                dis.Dispose();
+                if (!caughtException)
+                    fail("Test failed");
+            }
+
+            // Ensure we get an IndexOutOfRangeException exception when length is negative
+            @in = GetType().Assembly.GetManifestResourceStream(READFULLY_TEST_FILE);
+            dis = new DataInputStream(@in);
+            caughtException = false;
+            try
+            {
+                dis.ReadFully(buffer, 0, -20);
+            }
+#pragma warning disable 168
+            catch (IndexOutOfRangeException ie)
+#pragma warning restore 168
+            {
+                caughtException = true;
+            }
+            finally
+            {
+                dis.Dispose();
+                if (!caughtException)
+                    fail("Test failed");
+            }
+        }
+
+        [Test]
+        public void TestReadLinePushback()
+        {
+            using (MemoryStream pis = new MemoryStream("\r".GetBytes(Encoding.UTF8)))
+            {
+                DataInputStream dis = new DataInputStream(pis);
+
+#pragma warning disable 612, 618
+                string line = dis.ReadLine();
+#pragma warning restore 612, 618
+                if (line == null)
+                {
+                    fail("Got null, should return empty line");
+                }
+
+                long count = pis.Length - (line.Length + 1 /*account for the newline*/);
+
+                if (count != 0)
+                {
+                    fail("Test failed: available() returns "
+                                         + count + " when the file is empty");
+                }
+            }
+        }
+
+        [Test]
+        public void TestReadUTF()
+        {
+            for (int i = 0; i < TEST_ITERATIONS; i++)
+            {
+                try
+                {
+                    WriteAndReadAString();
+                }
+                catch (FormatException utfdfe)
+                {
+                    if (utfdfe.Message == null)
+                        fail("vague exception thrown");
+                }
+#pragma warning disable 168
+                catch (EndOfStreamException eofe)
+#pragma warning restore 168
+                {
+                    // These are rare and beyond the scope of the test
+                }
+            }
+        }
+
+
+        private static readonly int TEST_ITERATIONS = 1000;
+
+        private static readonly int A_NUMBER_NEAR_65535 = 60000;
+
+        private static readonly int MAX_CORRUPTIONS_PER_CYCLE = 3;
+
+        private static void WriteAndReadAString()
+        {
+            // Write out a string whose UTF-8 encoding is quite possibly
+            // longer than 65535 bytes
+            int length = Random().nextInt(A_NUMBER_NEAR_65535) + 1;
+            MemoryStream baos = new MemoryStream();
+            StringBuilder testBuffer = new StringBuilder();
+            for (int i = 0; i < length; i++)
+                testBuffer.append((char)Random().Next());
+            string testString = testBuffer.toString();
+            DataOutputStream dos = new DataOutputStream(baos);
+            dos.WriteUTF(testString);
+
+            // Corrupt the data to produce malformed characters
+            byte[] testBytes = baos.ToArray();
+            int dataLength = testBytes.Length;
+            int corruptions = Random().nextInt(MAX_CORRUPTIONS_PER_CYCLE);
+            for (int i = 0; i < corruptions; i++)
+            {
+                int index = Random().nextInt(dataLength);
+                testBytes[index] = (byte)Random().Next();
+            }
+
+            // Pay special attention to mangling the end to produce
+            // partial characters at end
+            testBytes[dataLength - 1] = (byte)Random().Next();
+            testBytes[dataLength - 2] = (byte)Random().Next();
+
+            // Attempt to decode the bytes back into a String
+            MemoryStream bais = new MemoryStream(testBytes);
+            DataInputStream dis = new DataInputStream(bais);
+            dis.ReadUTF();
+        }
+
+        [Test]
+        public void TestSkipBytes()
+        {
+            DataInputStream dis = new DataInputStream(new MyInputStream());
+            dotest(dis, 0, 11, -1, 0);
+            dotest(dis, 0, 11, 5, 5);
+            Console.WriteLine("\n***CAUTION**** - may go into an infinite loop");
+            dotest(dis, 5, 11, 20, 6);
+        }
+
+
+        private static void dotest(DataInputStream dis, int pos, int total,
+                               int toskip, int expected)
+        {
+
+            try
+            {
+                if (VERBOSE)
+                {
+                    Console.WriteLine("\n\nTotal bytes in the stream = " + total);
+                    Console.WriteLine("Currently at position = " + pos);
+                    Console.WriteLine("Bytes to skip = " + toskip);
+                    Console.WriteLine("Expected result = " + expected);
+                }
+                int skipped = dis.SkipBytes(toskip);
+                if (VERBOSE)
+                {
+                    Console.WriteLine("Actual skipped = " + skipped);
+                }
+                if (skipped != expected)
+                {
+                    fail("DataInputStream.skipBytes does not return expected value");
+                }
+            }
+            catch (EndOfStreamException e)
+            {
+                fail("DataInputStream.skipBytes throws unexpected EOFException");
+            }
+            catch (IOException e)
+            {
+                Console.WriteLine("IOException is thrown - possible result");
+            }
+        }
+
+        internal class MyInputStream : MemoryStream
+        {
+
+            private int readctr = 0;
+
+
+            public override int ReadByte()
+            {
+
+                if (readctr > 10)
+                {
+                    return -1;
+                }
+                else
+                {
+                    readctr++;
+                    return 0;
+                }
+
+            }
+
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/dec66102/src/Lucene.Net.Tests/core/Support/TestDataOutputStream.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/core/Support/TestDataOutputStream.cs b/src/Lucene.Net.Tests/core/Support/TestDataOutputStream.cs
new file mode 100644
index 0000000..11c1ce5
--- /dev/null
+++ b/src/Lucene.Net.Tests/core/Support/TestDataOutputStream.cs
@@ -0,0 +1,172 @@
+using Lucene.Net.Util;
+using NUnit.Framework;
+using System;
+using System.Diagnostics;
+using System.IO;
+
+namespace Lucene.Net.Support
+{
+    public class TestDataOutputStream : LuceneTestCase
+    {
+        [Test]
+        public void TestCounterOverflow()
+        {
+            var output = new MemoryStream();
+            CounterOverflow dataOut = new CounterOverflow(output);
+
+            dataOut.WriteByte(1);
+            if (dataOut.Length < 0)
+            {
+                fail("Internal counter less than 0.");
+            }
+        }
+
+        private class CounterOverflow : DataOutputStream
+        {
+            public CounterOverflow(Stream output)
+                : base(output)
+            {
+                base.written = int.MaxValue;
+            }
+        }
+
+        [Test]
+        public void TestWriteUTF()
+        {
+            ByteArrayOutputStream baos = new ByteArrayOutputStream();
+            DataOutputStream dos = new DataOutputStream(baos);
+            dos.WriteUTF("Hello, World!");  // 15
+            dos.Flush();
+            if (baos.Length != dos.Length)
+                fail("Miscounted bytes in DataOutputStream.");
+        }
+
+        [Test]
+        public void TestBoundsCheck()
+        {
+            byte[] data = { 90, 91, 92, 93, 94, 95, 96, 97, 98, 99 };
+            ByteArrayOutputStream bos = new ByteArrayOutputStream();
+            DummyFilterStream dfs = new DummyFilterStream(bos);
+            bool caughtException = false;
+
+            // -ve length
+            try
+            {
+                dfs.Write(data, 0, -5);
+            }
+#pragma warning disable 168
+            catch (ArgumentOutOfRangeException ie)
+#pragma warning restore 168
+            {
+                caughtException = true;
+            }
+            finally
+            {
+                if (!caughtException)
+                    fail("Test failed");
+            }
+
+            // -ve offset
+            caughtException = false;
+            try
+            {
+                dfs.Write(data, -2, 5);
+            }
+#pragma warning disable 168
+            catch (ArgumentOutOfRangeException ie)
+#pragma warning restore 168
+            {
+                caughtException = true;
+            }
+            finally
+            {
+                if (!caughtException)
+                    fail("Test failed");
+            }
+
+            // off + len > data.length
+            caughtException = false;
+            try
+            {
+                dfs.Write(data, 6, 5);
+            }
+#pragma warning disable 168
+            catch (ArgumentException ie)
+#pragma warning restore 168
+            {
+                caughtException = true;
+            }
+            finally
+            {
+                if (!caughtException)
+                    fail("Test failed");
+            }
+
+            // null data
+            caughtException = false;
+            try
+            {
+                dfs.Write(null, 0, 5);
+            }
+#pragma warning disable 168
+            catch (ArgumentNullException re)
+#pragma warning restore 168
+            {
+                caughtException = true;
+            }
+            finally
+            {
+                if (!caughtException)
+                    fail("Test failed");
+            }
+        }
+
+        private class DummyFilterStream : DataOutputStream
+        {
+
+            public DummyFilterStream(Stream o)
+                    : base(o)
+            {
+            }
+
+            public override void Write(int val)
+            {
+                base.Write(val + 1);
+            }
+        }
+
+        [Test]
+        public void TestWrite()
+        {
+            IDataOutput f = new F(new Sink());
+            f.Write(new byte[] { 1, 2, 3 }, 0, 3);
+        }
+
+        private class F : DataOutputStream
+        {
+
+            public F(Stream o)
+                    : base(o)
+            {
+            }
+
+            public override void Write(int b)
+            {
+                Debug.WriteLine("Ignoring write of " + b);
+            }
+
+        }
+
+        private class Sink : MemoryStream
+        {
+
+            public override void WriteByte(byte b)
+            {
+                throw new Exception("DataOutputStream directly invoked"
+                                           + " Write(byte) method of underlying"
+                                           + " stream");
+            }
+
+        }
+    }
+}


Mime
View raw message