lucenenet-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From synhers...@apache.org
Subject lucenenet git commit: LUCENET-555 Removing dependency on external zip library.
Date Fri, 25 Mar 2016 08:19:31 GMT
Repository: lucenenet
Updated Branches:
  refs/heads/master c6d7bec47 -> e218ff63d


LUCENET-555 Removing dependency on external zip library.

Adding ignore for project.lock.json files.


Project: http://git-wip-us.apache.org/repos/asf/lucenenet/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucenenet/commit/e218ff63
Tree: http://git-wip-us.apache.org/repos/asf/lucenenet/tree/e218ff63
Diff: http://git-wip-us.apache.org/repos/asf/lucenenet/diff/e218ff63

Branch: refs/heads/master
Commit: e218ff63d0b22cbe39543986f0382f241cf47c24
Parents: c6d7bec
Author: Elizabeth Maher <elizabeth.maher@microsoft.com>
Authored: Thu Mar 24 15:42:32 2016 -0700
Committer: Elizabeth Maher <elizabeth.maher@microsoft.com>
Committed: Thu Mar 24 19:07:22 2016 -0700

----------------------------------------------------------------------
 .gitignore                                      |   3 +-
 .../Codecs/Compressing/CompressionMode.cs       | 106 +++++++------------
 .../Document/CompressionTools.cs                |  80 +++++---------
 src/Lucene.Net.Core/Lucene.Net.csproj           |   3 -
 src/Lucene.Net.TestFramework/Util/TestUtil.cs   |  96 -----------------
 5 files changed, 70 insertions(+), 218 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucenenet/blob/e218ff63/.gitignore
----------------------------------------------------------------------
diff --git a/.gitignore b/.gitignore
index a1ea325..27e2d1b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -11,6 +11,7 @@ obj
 *.userprefs
 *.pidb
 *.ide
+*.project.lock.json
 test-results
 build/artifacts
 build/bin
@@ -31,4 +32,4 @@ bin/
 obj/
 doc/
 src/demo/
-packages/
\ No newline at end of file
+packages/

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/e218ff63/src/Lucene.Net.Core/Codecs/Compressing/CompressionMode.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Core/Codecs/Compressing/CompressionMode.cs b/src/Lucene.Net.Core/Codecs/Compressing/CompressionMode.cs
index be238b3..f319483 100644
--- a/src/Lucene.Net.Core/Codecs/Compressing/CompressionMode.cs
+++ b/src/Lucene.Net.Core/Codecs/Compressing/CompressionMode.cs
@@ -1,4 +1,6 @@
 using System.Diagnostics;
+using System.IO;
+using System.IO.Compression;
 
 namespace Lucene.Net.Codecs.Compressing
 {
@@ -81,7 +83,7 @@ namespace Lucene.Net.Codecs.Compressing
 
             public override Compressor NewCompressor()
             {
-                return new DeflateCompressor(Deflater.BEST_COMPRESSION);
+                return new DeflateCompressor(System.IO.Compression.CompressionLevel.Optimal);
             }
 
             public override Decompressor NewDecompressor()
@@ -204,16 +206,12 @@ namespace Lucene.Net.Codecs.Compressing
 
         private sealed class DeflateDecompressor : Decompressor
         {
-            internal readonly Inflater decompressor;
-            internal byte[] Compressed;
 
             internal DeflateDecompressor()
             {
-                decompressor = SharpZipLib.CreateInflater();
-                Compressed = new byte[0];
             }
 
-            public override void Decompress(DataInput @in, int originalLength, int offset,
int length, BytesRef bytes)
+            public override void Decompress(DataInput input, int originalLength, int offset,
int length, BytesRef bytes)
             {
                 Debug.Assert(offset + length <= originalLength);
                 if (length == 0)
@@ -221,45 +219,31 @@ namespace Lucene.Net.Codecs.Compressing
                     bytes.Length = 0;
                     return;
                 }
-                int compressedLength = @in.ReadVInt();
-                if (compressedLength > Compressed.Length)
-                {
-                    Compressed = new byte[ArrayUtil.Oversize(compressedLength, 1)];
-                }
-                @in.ReadBytes(Compressed, 0, compressedLength);
 
-                decompressor.Reset();
-                decompressor.SetInput(Compressed, 0, compressedLength);
+                byte[] compressedBytes = new byte[input.ReadVInt()];
+                input.ReadBytes(compressedBytes, 0, compressedBytes.Length);
+                byte[] decompressedBytes = null;
 
-                bytes.Offset = bytes.Length = 0;
-                while (true)
+                using (MemoryStream decompressedStream = new MemoryStream())
                 {
-                    int count;
-                    try
-                    {
-                        int remaining = bytes.Bytes.Length - bytes.Length;
-                        count = decompressor.Inflate((byte[])(Array)(bytes.Bytes), bytes.Length,
remaining);
-                    }
-                    catch (System.FormatException e)
-                    {
-                        throw new System.IO.IOException("See inner", e);
-                    }
-                    bytes.Length += count;
-                    if (decompressor.IsFinished)
-                    {
-                        break;
-                    }
-                    else
+                    using (MemoryStream compressedStream = new MemoryStream(compressedBytes))
                     {
-                        bytes.Bytes = ArrayUtil.Grow(bytes.Bytes);
+                        using (DeflateStream dStream = new DeflateStream(compressedStream,
System.IO.Compression.CompressionMode.Decompress))
+                        {
+                            dStream.CopyTo(decompressedStream);
+                        }
                     }
+                    decompressedBytes = decompressedStream.ToArray();
                 }
-                if (bytes.Length != originalLength)
+
+                if (decompressedBytes.Length != originalLength)
                 {
-                    throw new CorruptIndexException("Lengths mismatch: " + bytes.Length +
" != " + originalLength + " (resource=" + @in + ")");
+                    throw new CorruptIndexException($"Length mismatch: {decompressedBytes.Length}
!= {originalLength} (resource={input})");
                 }
+
+                bytes.Bytes = decompressedBytes;
                 bytes.Offset = offset;
-                bytes.Length = length;
+                bytes.Length = length;            
             }
 
             public override object Clone()
@@ -270,48 +254,36 @@ namespace Lucene.Net.Codecs.Compressing
 
         private class DeflateCompressor : Compressor
         {
-            private readonly Deflater Compressor;
-            private byte[] Compressed;
-
-            internal DeflateCompressor(int level)
+            private CompressionLevel compressionLevel;
+            internal DeflateCompressor(CompressionLevel level)
             {
-                Compressor = SharpZipLib.CreateDeflater();
-                Compressed = new byte[64];
+                compressionLevel = level;
             }
 
-            public override void Compress(byte[] bytes, int off, int len, DataOutput @out)
+            public override void Compress(byte[] bytes, int off, int len, DataOutput output)
             {
-                Compressor.Reset();
-                Compressor.SetInput((byte[])(Array)bytes, off, len);
-                Compressor.Finish();
+                byte[] resultArray = null;
+                using (MemoryStream compressionMemoryStream = new MemoryStream())
+                {
+                    using (DeflateStream deflateStream = new DeflateStream(compressionMemoryStream,
compressionLevel))
+                    {
+                        deflateStream.Write(bytes, off, len);
+                    }
+                    resultArray = compressionMemoryStream.ToArray();
+                }
 
-                if (Compressor.NeedsInput)
+                if (resultArray.Length == 0)
                 {
-                    // no output
                     Debug.Assert(len == 0, len.ToString());
-                    @out.WriteVInt(0);
+                    output.WriteVInt(0);
                     return;
                 }
-
-                int totalCount = 0;
-                for (; ; )
+                else
                 {
-                    int count = Compressor.Deflate(Compressed, totalCount, Compressed.Length
- totalCount);
-                    totalCount += count;
-                    Debug.Assert(totalCount <= Compressed.Length);
-                    if (Compressor.IsFinished)
-                    {
-                        break;
-                    }
-                    else
-                    {
-                        Compressed = ArrayUtil.Grow(Compressed);
-                    }
+                    output.WriteVInt(resultArray.Length);
+                    output.WriteBytes(resultArray, resultArray.Length);
                 }
-
-                @out.WriteVInt(totalCount);
-                @out.WriteBytes(Compressed, totalCount);
             }
         }
     }
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/e218ff63/src/Lucene.Net.Core/Document/CompressionTools.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Core/Document/CompressionTools.cs b/src/Lucene.Net.Core/Document/CompressionTools.cs
index d5e369d..b7e72b6 100644
--- a/src/Lucene.Net.Core/Document/CompressionTools.cs
+++ b/src/Lucene.Net.Core/Document/CompressionTools.cs
@@ -1,4 +1,6 @@
 using System;
+using System.IO;
+using System.IO.Compression;
 using Lucene.Net.Support;
 using Lucene.Net.Util;
 
@@ -37,67 +39,50 @@ namespace Lucene.Net.Documents
 
         /// <summary>
         /// Compresses the specified byte range using the
-        ///  specified compressionLevel (constants are defined in
-        ///  java.util.zip.Deflater).
+        ///  specified compressionLevel 
         /// </summary>
-        public static byte[] Compress(byte[] value, int offset, int length, int compressionLevel)
+        public static byte[] Compress(byte[] value, int offset, int length, CompressionLevel
compressionLevel)
         {
-            /* Create an expandable byte array to hold the compressed data.
-             * You cannot use an array that's the same size as the orginal because
-             * there is no guarantee that the compressed data will be smaller than
-             * the uncompressed data. */
-            var bos = new ByteArrayOutputStream(length);
-
-            Deflater compressor = SharpZipLib.CreateDeflater();
-
-            try
+            byte[] resultArray = null;
+            using (MemoryStream compressionMemoryStream = new MemoryStream())
             {
-                compressor.SetLevel(compressionLevel);
-                compressor.SetInput(value, offset, length);
-                compressor.Finish();
-
-                // Compress the data
-                var buf = new byte[1024];
-                while (!compressor.IsFinished)
+                using (DeflateStream deflateStream = new DeflateStream(compressionMemoryStream,
compressionLevel))
                 {
-                    int count = compressor.Deflate(buf);
-                    bos.Write(buf, 0, count);
+
+                    deflateStream.Write(value, offset, length);
                 }
+                resultArray = compressionMemoryStream.ToArray();
             }
-            finally
-            {
-            }
-
-            return bos.ToArray();
+            return resultArray;
         }
 
         /// <summary>
-        /// Compresses the specified byte range, with default BEST_COMPRESSION level </summary>
+        /// Compresses the specified byte range, with default Optimal level 
+        /// </summary>
         public static byte[] Compress(byte[] value, int offset, int length)
         {
-            return Compress(value, offset, length, Deflater.BEST_COMPRESSION);
+            return Compress(value, offset, length, CompressionLevel.Optimal);
         }
 
         /// <summary>
-        /// Compresses all bytes in the array, with default BEST_COMPRESSION level </summary>
+        /// Compresses all bytes in the array, with default Optimal level </summary>
         public static byte[] Compress(byte[] value)
         {
-            return Compress(value, 0, value.Length, Deflater.BEST_COMPRESSION);
+            return Compress(value, 0, value.Length, CompressionLevel.Optimal);
         }
 
         /// <summary>
         /// Compresses the String value, with default BEST_COMPRESSION level </summary>
         public static byte[] CompressString(string value)
         {
-            return CompressString(value, Deflater.BEST_COMPRESSION);
+            return CompressString(value, CompressionLevel.Optimal);
         }
 
         /// <summary>
         /// Compresses the String value using the specified
-        ///  compressionLevel (constants are defined in
-        ///  java.util.zip.Deflater).
+        ///  compressionLevel.
         /// </summary>
-        public static byte[] CompressString(string value, int compressionLevel)
+        public static byte[] CompressString(string value, CompressionLevel compressionLevel)
         {
             var result = new BytesRef();
             UnicodeUtil.UTF16toUTF8(value.ToCharArray(), 0, value.Length, result);
@@ -128,28 +113,21 @@ namespace Lucene.Net.Documents
         /// </summary>
         public static byte[] Decompress(byte[] value, int offset, int length)
         {
-            // Create an expandable byte array to hold the decompressed data
-            var bos = new ByteArrayOutputStream(length);
-
-            Inflater decompressor = SharpZipLib.CreateInflater();
+            byte[] decompressedBytes = null;
 
-            try
+            using (MemoryStream decompressedStream = new MemoryStream())
             {
-                decompressor.SetInput(value);
-
-                // Decompress the data
-                var buf = new byte[1024];
-                while (!decompressor.IsFinished)
+                using (MemoryStream compressedStream = new MemoryStream(value))
                 {
-                    int count = decompressor.Inflate(buf);
-                    bos.Write(buf, 0, count);
+                    using (DeflateStream dStream = new DeflateStream(compressedStream, CompressionMode.Decompress))
+                    {
+                        dStream.CopyTo(decompressedStream);
+                    }
                 }
-            }
-            finally
-            {
+                decompressedBytes = decompressedStream.ToArray();
             }
 
-            return bos.ToArray();
+            return decompressedBytes;
         }
 
         /// <summary>
@@ -182,4 +160,4 @@ namespace Lucene.Net.Documents
             return DecompressString(bytes.Bytes, bytes.Offset, bytes.Length);
         }
     }
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/e218ff63/src/Lucene.Net.Core/Lucene.Net.csproj
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Core/Lucene.Net.csproj b/src/Lucene.Net.Core/Lucene.Net.csproj
index 7d3e2c1..5d573a0 100644
--- a/src/Lucene.Net.Core/Lucene.Net.csproj
+++ b/src/Lucene.Net.Core/Lucene.Net.csproj
@@ -40,9 +40,6 @@
     <StartupObject />
   </PropertyGroup>
   <ItemGroup>
-    <Reference Include="ICSharpCode.SharpZipLib">
-      <HintPath>..\..\packages\SharpZipLib.0.86.0\lib\20\ICSharpCode.SharpZipLib.dll</HintPath>
-    </Reference>
     <Reference Include="Microsoft.CSharp" />
     <Reference Include="System" />
     <Reference Include="System.Collections.Immutable">

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/e218ff63/src/Lucene.Net.TestFramework/Util/TestUtil.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.TestFramework/Util/TestUtil.cs b/src/Lucene.Net.TestFramework/Util/TestUtil.cs
index 7250cdb..1afcf67 100644
--- a/src/Lucene.Net.TestFramework/Util/TestUtil.cs
+++ b/src/Lucene.Net.TestFramework/Util/TestUtil.cs
@@ -1,5 +1,4 @@
 using System.Numerics;
-using ICSharpCode.SharpZipLib.Zip;
 using Lucene.Net.Documents;
 using Lucene.Net.Index;
 using System;
@@ -124,101 +123,6 @@ namespace Lucene.Net.Util
             return unremoved;
         }
 
-        /// <summary>
-        /// Convenience method unzipping zipName into destDir, cleaning up
-        /// destDir first.
-        /// </summary>
-        public static void Unzip(FileInfo zipName, DirectoryInfo destDir)
-        {
-            System.IO.Directory.Delete(destDir.FullName, true);
-            destDir.CreateSubdirectory(destDir.FullName);// mkdir();
-
-            using (ZipInputStream s = new ZipInputStream(File.OpenRead(zipName.FullName)))
-            {
-                ZipEntry entry;
-                while ((entry = s.GetNextEntry()) != null)
-                {
-                    string directoryName = Path.GetDirectoryName(entry.Name); //Directory
where file is stored
-                    string fileName = Path.GetFileName(entry.Name); //Name of file (if it
is one)
-                    DirectoryInfo targetDir = new DirectoryInfo(Path.Combine(destDir.FullName,
entry.Name));
-                    if (entry.IsDirectory)
-                    {
-                        targetDir.Create();
-                    }
-                    else
-                    {
-                        if (targetDir.Parent != null)
-                        {
-                            // be on the safe side: do not rely on that directories are always
extracted
-                            // before their children (although this makes sense, but is it
guaranteed?)
-                            targetDir.Parent.Create();
-                        }
-
-                        if (fileName != String.Empty)
-                        {
-                            using (FileStream streamWriter = File.Create(entry.Name))
-                            {
-                                int size = 2048;
-                                byte[] data = new byte[2048];
-                                while (true)
-                                {
-                                    size = s.Read(data, 0, data.Length);
-                                    if (size > 0)
-                                    {
-                                        streamWriter.Write(data, 0, size);
-                                    }
-                                    else
-                                    {
-                                        break;
-                                    }
-                                }
-                            }
-                        }
-                    }
-                }
-            }
-
-            /*
-           ZipFile zipFile = new ZipFile(zipName);
-           IEnumerator<FileInfo> entries = zipFile.entries();
-
-           while (entries.MoveNext())
-           {
-             ZipEntry entry = entries.Current;
-
-             Stream @in = zipFile.getInputStream(entry);
-             DirectoryInfo targetFile = new DirectoryInfo(destDir, entry.Name);
-             if (entry.Directory)
-             {
-               // allow unzipping with directory structure
-               //targetFile.mkdirs();
-                 targetFile.Create();
-             }
-             else
-             {
-               if (targetFile.ParentFile != null)
-               {
-                 // be on the safe side: do not rely on that directories are always extracted
-                 // before their children (although this makes sense, but is it guaranteed?)
-                 targetFile.ParentFile.mkdirs();
-               }
-               Stream @out = new BufferedOutputStream(new FileOutputStream(targetFile));
-
-               byte[] buffer = new byte[8192];
-               int len;
-               while ((len = @in.Read(buffer, 0, buffer.Length)) >= 0)
-               {
-                 @out.Write(buffer, 0, len);
-               }
-
-               @in.Close();
-               @out.Close();
-             }
-           }
-
-           zipFile.close();*/
-        }
-
         public static void SyncConcurrentMerges(IndexWriter writer)
         {
             SyncConcurrentMerges(writer.Config.MergeScheduler);


Mime
View raw message