lucenenet-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From nightowl...@apache.org
Subject [45/62] [abbrv] [partial] lucenenet git commit: Renamed Lucene.Net.Core folder Lucene.Net because the dotnet.exe pack command doesn't allow creating a NuGet package with a different name than its folder. Working around it with the script was much more co
Date Tue, 04 Apr 2017 17:19:51 GMT
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/a5dc68d0/src/Lucene.Net.Core/Codecs/Compressing/CompressingStoredFieldsIndexReader.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Core/Codecs/Compressing/CompressingStoredFieldsIndexReader.cs b/src/Lucene.Net.Core/Codecs/Compressing/CompressingStoredFieldsIndexReader.cs
deleted file mode 100644
index a27fc40..0000000
--- a/src/Lucene.Net.Core/Codecs/Compressing/CompressingStoredFieldsIndexReader.cs
+++ /dev/null
@@ -1,210 +0,0 @@
-using Lucene.Net.Support;
-using ArrayUtil = Lucene.Net.Util.ArrayUtil;
-
-namespace Lucene.Net.Codecs.Compressing
-{
-    /*
-     * Licensed to the Apache Software Foundation (ASF) under one or more
-     * contributor license agreements.  See the NOTICE file distributed with
-     * this work for additional information regarding copyright ownership.
-     * The ASF licenses this file to You under the Apache License, Version 2.0
-     * (the "License"); you may not use this file except in compliance with
-     * the License.  You may obtain a copy of the License at
-     *
-     *     http://www.apache.org/licenses/LICENSE-2.0
-     *
-     * Unless required by applicable law or agreed to in writing, software
-     * distributed under the License is distributed on an "AS IS" BASIS,
-     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-     * See the License for the specific language governing permissions and
-     * limitations under the License.
-     */
-
-    using CorruptIndexException = Lucene.Net.Index.CorruptIndexException;
-    using IndexInput = Lucene.Net.Store.IndexInput;
-    using PackedInt32s = Lucene.Net.Util.Packed.PackedInt32s;
-    using RamUsageEstimator = Lucene.Net.Util.RamUsageEstimator;
-    using SegmentInfo = Lucene.Net.Index.SegmentInfo;
-
-    /// <summary>
-    /// Random-access reader for <seealso cref="CompressingStoredFieldsIndexWriter"/>.
-    /// @lucene.internal
-    /// </summary>
-    public sealed class CompressingStoredFieldsIndexReader
-    {
-        internal static long MoveLowOrderBitToSign(long n)
-        {
-            return (((long)((ulong)n >> 1)) ^ -(n & 1));
-        }
-
-        internal readonly int maxDoc;
-        internal readonly int[] docBases;
-        internal readonly long[] startPointers;
-        internal readonly int[] avgChunkDocs;
-        internal readonly long[] avgChunkSizes;
-        internal readonly PackedInt32s.Reader[] docBasesDeltas; // delta from the avg
-        internal readonly PackedInt32s.Reader[] startPointersDeltas; // delta from the avg
-
-        // It is the responsibility of the caller to close fieldsIndexIn after this constructor
-        // has been called
-        internal CompressingStoredFieldsIndexReader(IndexInput fieldsIndexIn, SegmentInfo si)
-        {
-            maxDoc = si.DocCount;
-            int[] docBases = new int[16];
-            long[] startPointers = new long[16];
-            int[] avgChunkDocs = new int[16];
-            long[] avgChunkSizes = new long[16];
-            PackedInt32s.Reader[] docBasesDeltas = new PackedInt32s.Reader[16];
-            PackedInt32s.Reader[] startPointersDeltas = new PackedInt32s.Reader[16];
-
-            int packedIntsVersion = fieldsIndexIn.ReadVInt32();
-
-            int blockCount = 0;
-
-            for (; ; )
-            {
-                int numChunks = fieldsIndexIn.ReadVInt32();
-                if (numChunks == 0)
-                {
-                    break;
-                }
-                if (blockCount == docBases.Length)
-                {
-                    int newSize = ArrayUtil.Oversize(blockCount + 1, 8);
-                    docBases = Arrays.CopyOf(docBases, newSize);
-                    startPointers = Arrays.CopyOf(startPointers, newSize);
-                    avgChunkDocs = Arrays.CopyOf(avgChunkDocs, newSize);
-                    avgChunkSizes = Arrays.CopyOf(avgChunkSizes, newSize);
-                    docBasesDeltas = Arrays.CopyOf(docBasesDeltas, newSize);
-                    startPointersDeltas = Arrays.CopyOf(startPointersDeltas, newSize);
-                }
-
-                // doc bases
-                docBases[blockCount] = fieldsIndexIn.ReadVInt32();
-                avgChunkDocs[blockCount] = fieldsIndexIn.ReadVInt32();
-                int bitsPerDocBase = fieldsIndexIn.ReadVInt32();
-                if (bitsPerDocBase > 32)
-                {
-                    throw new CorruptIndexException("Corrupted bitsPerDocBase (resource=" + fieldsIndexIn + ")");
-                }
-                docBasesDeltas[blockCount] = PackedInt32s.GetReaderNoHeader(fieldsIndexIn, PackedInt32s.Format.PACKED, packedIntsVersion, numChunks, bitsPerDocBase);
-
-                // start pointers
-                startPointers[blockCount] = fieldsIndexIn.ReadVInt64();
-                avgChunkSizes[blockCount] = fieldsIndexIn.ReadVInt64();
-                int bitsPerStartPointer = fieldsIndexIn.ReadVInt32();
-                if (bitsPerStartPointer > 64)
-                {
-                    throw new CorruptIndexException("Corrupted bitsPerStartPointer (resource=" + fieldsIndexIn + ")");
-                }
-                startPointersDeltas[blockCount] = PackedInt32s.GetReaderNoHeader(fieldsIndexIn, PackedInt32s.Format.PACKED, packedIntsVersion, numChunks, bitsPerStartPointer);
-
-                ++blockCount;
-            }
-
-            this.docBases = Arrays.CopyOf(docBases, blockCount);
-            this.startPointers = Arrays.CopyOf(startPointers, blockCount);
-            this.avgChunkDocs = Arrays.CopyOf(avgChunkDocs, blockCount);
-            this.avgChunkSizes = Arrays.CopyOf(avgChunkSizes, blockCount);
-            this.docBasesDeltas = Arrays.CopyOf(docBasesDeltas, blockCount);
-            this.startPointersDeltas = Arrays.CopyOf(startPointersDeltas, blockCount);
-        }
-
-        private int Block(int docID)
-        {
-            int lo = 0, hi = docBases.Length - 1;
-            while (lo <= hi)
-            {
-                int mid = (int)((uint)(lo + hi) >> 1);
-                int midValue = docBases[mid];
-                if (midValue == docID)
-                {
-                    return mid;
-                }
-                else if (midValue < docID)
-                {
-                    lo = mid + 1;
-                }
-                else
-                {
-                    hi = mid - 1;
-                }
-            }
-            return hi;
-        }
-
-        private int RelativeDocBase(int block, int relativeChunk)
-        {
-            int expected = avgChunkDocs[block] * relativeChunk;
-            long delta = MoveLowOrderBitToSign(docBasesDeltas[block].Get(relativeChunk));
-            return expected + (int)delta;
-        }
-
-        private long RelativeStartPointer(int block, int relativeChunk)
-        {
-            long expected = avgChunkSizes[block] * relativeChunk;
-            long delta = MoveLowOrderBitToSign(startPointersDeltas[block].Get(relativeChunk));
-            return expected + delta;
-        }
-
-        private int RelativeChunk(int block, int relativeDoc)
-        {
-            int lo = 0, hi = docBasesDeltas[block].Count - 1;
-            while (lo <= hi)
-            {
-                int mid = (int)((uint)(lo + hi) >> 1);
-                int midValue = RelativeDocBase(block, mid);
-                if (midValue == relativeDoc)
-                {
-                    return mid;
-                }
-                else if (midValue < relativeDoc)
-                {
-                    lo = mid + 1;
-                }
-                else
-                {
-                    hi = mid - 1;
-                }
-            }
-            return hi;
-        }
-
-        internal long GetStartPointer(int docID)
-        {
-            if (docID < 0 || docID >= maxDoc)
-            {
-                throw new System.ArgumentException("docID out of range [0-" + maxDoc + "]: " + docID);
-            }
-            int block = Block(docID);
-            int relativeChunk = RelativeChunk(block, docID - docBases[block]);
-            return startPointers[block] + RelativeStartPointer(block, relativeChunk);
-        }
-
-        public object Clone()
-        {
-            return this;
-        }
-
-        internal long RamBytesUsed()
-        {
-            long res = 0;
-
-            foreach (PackedInt32s.Reader r in docBasesDeltas)
-            {
-                res += r.RamBytesUsed();
-            }
-            foreach (PackedInt32s.Reader r in startPointersDeltas)
-            {
-                res += r.RamBytesUsed();
-            }
-
-            res += RamUsageEstimator.SizeOf(docBases);
-            res += RamUsageEstimator.SizeOf(startPointers);
-            res += RamUsageEstimator.SizeOf(avgChunkDocs);
-            res += RamUsageEstimator.SizeOf(avgChunkSizes);
-
-            return res;
-        }
-    }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/a5dc68d0/src/Lucene.Net.Core/Codecs/Compressing/CompressingStoredFieldsIndexWriter.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Core/Codecs/Compressing/CompressingStoredFieldsIndexWriter.cs b/src/Lucene.Net.Core/Codecs/Compressing/CompressingStoredFieldsIndexWriter.cs
deleted file mode 100644
index 981e476..0000000
--- a/src/Lucene.Net.Core/Codecs/Compressing/CompressingStoredFieldsIndexWriter.cs
+++ /dev/null
@@ -1,227 +0,0 @@
-using System;
-using System.Diagnostics;
-
-namespace Lucene.Net.Codecs.Compressing
-{
-    /*
-     * Licensed to the Apache Software Foundation (ASF) under one or more
-     * contributor license agreements.  See the NOTICE file distributed with
-     * this work for additional information regarding copyright ownership.
-     * The ASF licenses this file to You under the Apache License, Version 2.0
-     * (the "License"); you may not use this file except in compliance with
-     * the License.  You may obtain a copy of the License at
-     *
-     *     http://www.apache.org/licenses/LICENSE-2.0
-     *
-     * Unless required by applicable law or agreed to in writing, software
-     * distributed under the License is distributed on an "AS IS" BASIS,
-     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-     * See the License for the specific language governing permissions and
-     * limitations under the License.
-     */
-
-    using IndexOutput = Lucene.Net.Store.IndexOutput;
-    using PackedInt32s = Lucene.Net.Util.Packed.PackedInt32s;
-
-    /// <summary>
-    /// Efficient index format for block-based <seealso cref="Codec"/>s.
-    /// <p> this writer generates a file which can be loaded into memory using
-    /// memory-efficient data structures to quickly locate the block that contains
-    /// any document.
-    /// <p>In order to have a compact in-memory representation, for every block of
-    /// 1024 chunks, this index computes the average number of bytes per
-    /// chunk and for every chunk, only stores the difference between<ul>
-    /// <li>${chunk number} * ${average length of a chunk}</li>
-    /// <li>and the actual start offset of the chunk</li></ul></p>
-    /// <p>Data is written as follows:</p>
-    /// <ul>
-    /// <li>PackedIntsVersion, &lt;Block&gt;<sup>BlockCount</sup>, BlocksEndMarker</li>
-    /// <li>PackedIntsVersion --&gt; <seealso cref="PackedInt32s#VERSION_CURRENT"/> as a <seealso cref="DataOutput#writeVInt VInt"/></li>
-    /// <li>BlocksEndMarker --&gt; <tt>0</tt> as a <seealso cref="DataOutput#writeVInt VInt"/>, this marks the end of blocks since blocks are not allowed to start with <tt>0</tt></li>
-    /// <li>Block --&gt; BlockChunks, &lt;DocBases&gt;, &lt;StartPointers&gt;</li>
-    /// <li>BlockChunks --&gt; a <seealso cref="DataOutput#writeVInt VInt"/> which is the number of chunks encoded in the block</li>
-    /// <li>DocBases --&gt; DocBase, AvgChunkDocs, BitsPerDocBaseDelta, DocBaseDeltas</li>
-    /// <li>DocBase --&gt; first document ID of the block of chunks, as a <seealso cref="DataOutput#writeVInt VInt"/></li>
-    /// <li>AvgChunkDocs --&gt; average number of documents in a single chunk, as a <seealso cref="DataOutput#writeVInt VInt"/></li>
-    /// <li>BitsPerDocBaseDelta --&gt; number of bits required to represent a delta from the average using <a href="https://developers.google.com/protocol-buffers/docs/encoding#types">ZigZag encoding</a></li>
-    /// <li>DocBaseDeltas --&gt; <seealso cref="PackedInt32s packed"/> array of BlockChunks elements of BitsPerDocBaseDelta bits each, representing the deltas from the average doc base using <a href="https://developers.google.com/protocol-buffers/docs/encoding#types">ZigZag encoding</a>.</li>
-    /// <li>StartPointers --&gt; StartPointerBase, AvgChunkSize, BitsPerStartPointerDelta, StartPointerDeltas</li>
-    /// <li>StartPointerBase --&gt; the first start pointer of the block, as a <seealso cref="DataOutput#writeVLong VLong"/></li>
-    /// <li>AvgChunkSize --&gt; the average size of a chunk of compressed documents, as a <seealso cref="DataOutput#writeVLong VLong"/></li>
-    /// <li>BitsPerStartPointerDelta --&gt; number of bits required to represent a delta from the average using <a href="https://developers.google.com/protocol-buffers/docs/encoding#types">ZigZag encoding</a></li>
-    /// <li>StartPointerDeltas --&gt; <seealso cref="PackedInt32s packed"/> array of BlockChunks elements of BitsPerStartPointerDelta bits each, representing the deltas from the average start pointer using <a href="https://developers.google.com/protocol-buffers/docs/encoding#types">ZigZag encoding</a></li>
-    /// <li>Footer --&gt; <seealso cref="CodecUtil#writeFooter CodecFooter"/></li>
-    /// </ul>
-    /// <p>Notes</p>
-    /// <ul>
-    /// <li>For any block, the doc base of the n-th chunk can be restored with
-    /// <code>DocBase + AvgChunkDocs * n + DocBaseDeltas[n]</code>.</li>
-    /// <li>For any block, the start pointer of the n-th chunk can be restored with
-    /// <code>StartPointerBase + AvgChunkSize * n + StartPointerDeltas[n]</code>.</li>
-    /// <li>Once data is loaded into memory, you can lookup the start pointer of any
-    /// document by performing two binary searches: a first one based on the values
-    /// of DocBase in order to find the right block, and then inside the block based
-    /// on DocBaseDeltas (by reconstructing the doc bases for every chunk).</li>
-    /// </ul>
-    /// @lucene.internal
-    /// </summary>
-    public sealed class CompressingStoredFieldsIndexWriter : IDisposable
-    {
-        internal const int BLOCK_SIZE = 1024; // number of chunks to serialize at once
-
-        internal static long MoveSignToLowOrderBit(long n)
-        {
-            return (n >> 63) ^ (n << 1);
-        }
-
-        internal readonly IndexOutput fieldsIndexOut;
-        internal int totalDocs;
-        internal int blockDocs;
-        internal int blockChunks;
-        internal long firstStartPointer;
-        internal long maxStartPointer;
-        internal readonly int[] docBaseDeltas;
-        internal readonly long[] startPointerDeltas;
-
-        internal CompressingStoredFieldsIndexWriter(IndexOutput indexOutput)
-        {
-            this.fieldsIndexOut = indexOutput;
-            Reset();
-            totalDocs = 0;
-            docBaseDeltas = new int[BLOCK_SIZE];
-            startPointerDeltas = new long[BLOCK_SIZE];
-            fieldsIndexOut.WriteVInt32(PackedInt32s.VERSION_CURRENT);
-        }
-
-        private void Reset()
-        {
-            blockChunks = 0;
-            blockDocs = 0;
-            firstStartPointer = -1; // means unset
-        }
-
-        private void WriteBlock()
-        {
-            Debug.Assert(blockChunks > 0);
-            fieldsIndexOut.WriteVInt32(blockChunks);
-
-            // The trick here is that we only store the difference from the average start
-            // pointer or doc base, this helps save bits per value.
-            // And in order to prevent a few chunks that would be far from the average to
-            // raise the number of bits per value for all of them, we only encode blocks
-            // of 1024 chunks at once
-            // See LUCENE-4512
-
-            // doc bases
-            int avgChunkDocs;
-            if (blockChunks == 1)
-            {
-                avgChunkDocs = 0;
-            }
-            else
-            {
-                avgChunkDocs = (int)Math.Round((float)(blockDocs - docBaseDeltas[blockChunks - 1]) / (blockChunks - 1));
-            }
-            fieldsIndexOut.WriteVInt32(totalDocs - blockDocs); // docBase
-            fieldsIndexOut.WriteVInt32(avgChunkDocs);
-            int docBase = 0;
-            long maxDelta = 0;
-            for (int i = 0; i < blockChunks; ++i)
-            {
-                int delta = docBase - avgChunkDocs * i;
-                maxDelta |= MoveSignToLowOrderBit(delta);
-                docBase += docBaseDeltas[i];
-            }
-
-            int bitsPerDocBase = PackedInt32s.BitsRequired(maxDelta);
-            fieldsIndexOut.WriteVInt32(bitsPerDocBase);
-            PackedInt32s.Writer writer = PackedInt32s.GetWriterNoHeader(fieldsIndexOut, PackedInt32s.Format.PACKED, blockChunks, bitsPerDocBase, 1);
-            docBase = 0;
-            for (int i = 0; i < blockChunks; ++i)
-            {
-                long delta = docBase - avgChunkDocs * i;
-                Debug.Assert(PackedInt32s.BitsRequired(MoveSignToLowOrderBit(delta)) <= writer.BitsPerValue);
-                writer.Add(MoveSignToLowOrderBit(delta));
-                docBase += docBaseDeltas[i];
-            }
-            writer.Finish();
-
-            // start pointers
-            fieldsIndexOut.WriteVInt64(firstStartPointer);
-            long avgChunkSize;
-            if (blockChunks == 1)
-            {
-                avgChunkSize = 0;
-            }
-            else
-            {
-                avgChunkSize = (maxStartPointer - firstStartPointer) / (blockChunks - 1);
-            }
-            fieldsIndexOut.WriteVInt64(avgChunkSize);
-            long startPointer = 0;
-            maxDelta = 0;
-            for (int i = 0; i < blockChunks; ++i)
-            {
-                startPointer += startPointerDeltas[i];
-                long delta = startPointer - avgChunkSize * i;
-                maxDelta |= MoveSignToLowOrderBit(delta);
-            }
-
-            int bitsPerStartPointer = PackedInt32s.BitsRequired(maxDelta);
-            fieldsIndexOut.WriteVInt32(bitsPerStartPointer);
-            writer = PackedInt32s.GetWriterNoHeader(fieldsIndexOut, PackedInt32s.Format.PACKED, blockChunks, bitsPerStartPointer, 1);
-            startPointer = 0;
-            for (int i = 0; i < blockChunks; ++i)
-            {
-                startPointer += startPointerDeltas[i];
-                long delta = startPointer - avgChunkSize * i;
-                Debug.Assert(PackedInt32s.BitsRequired(MoveSignToLowOrderBit(delta)) <= writer.BitsPerValue);
-                writer.Add(MoveSignToLowOrderBit(delta));
-            }
-            writer.Finish();
-        }
-
-        internal void WriteIndex(int numDocs, long startPointer)
-        {
-            if (blockChunks == BLOCK_SIZE)
-            {
-                WriteBlock();
-                Reset();
-            }
-
-            if (firstStartPointer == -1)
-            {
-                firstStartPointer = maxStartPointer = startPointer;
-            }
-            Debug.Assert(firstStartPointer > 0 && startPointer >= firstStartPointer);
-
-            docBaseDeltas[blockChunks] = numDocs;
-            startPointerDeltas[blockChunks] = startPointer - maxStartPointer;
-
-            ++blockChunks;
-            blockDocs += numDocs;
-            totalDocs += numDocs;
-            maxStartPointer = startPointer;
-        }
-
-        internal void Finish(int numDocs, long maxPointer)
-        {
-            if (numDocs != totalDocs)
-            {
-                throw new Exception("Expected " + numDocs + " docs, but got " + totalDocs);
-            }
-            if (blockChunks > 0)
-            {
-                WriteBlock();
-            }
-            fieldsIndexOut.WriteVInt32(0); // end marker
-            fieldsIndexOut.WriteVInt64(maxPointer);
-            CodecUtil.WriteFooter(fieldsIndexOut);
-        }
-
-        public void Dispose()
-        {
-            fieldsIndexOut.Dispose();
-        }
-    }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/a5dc68d0/src/Lucene.Net.Core/Codecs/Compressing/CompressingStoredFieldsReader.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Core/Codecs/Compressing/CompressingStoredFieldsReader.cs b/src/Lucene.Net.Core/Codecs/Compressing/CompressingStoredFieldsReader.cs
deleted file mode 100644
index 335aa08..0000000
--- a/src/Lucene.Net.Core/Codecs/Compressing/CompressingStoredFieldsReader.cs
+++ /dev/null
@@ -1,638 +0,0 @@
-using Lucene.Net.Codecs.Lucene40;
-using Lucene.Net.Support;
-using System;
-using System.Diagnostics;
-using System.Reflection;
-
-namespace Lucene.Net.Codecs.Compressing
-{
-    /*
-     * Licensed to the Apache Software Foundation (ASF) under one or more
-     * contributor license agreements.  See the NOTICE file distributed with
-     * this work for additional information regarding copyright ownership.
-     * The ASF licenses this file to You under the Apache License, Version 2.0
-     * (the "License"); you may not use this file except in compliance with
-     * the License.  You may obtain a copy of the License at
-     *
-     *     http://www.apache.org/licenses/LICENSE-2.0
-     *
-     * Unless required by applicable law or agreed to in writing, software
-     * distributed under the License is distributed on an "AS IS" BASIS,
-     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-     * See the License for the specific language governing permissions and
-     * limitations under the License.
-     */
-
-    using ArrayUtil = Lucene.Net.Util.ArrayUtil;
-    using BufferedChecksumIndexInput = Lucene.Net.Store.BufferedChecksumIndexInput;
-    using ByteArrayDataInput = Lucene.Net.Store.ByteArrayDataInput;
-    using BytesRef = Lucene.Net.Util.BytesRef;
-    using ChecksumIndexInput = Lucene.Net.Store.ChecksumIndexInput;
-    using CorruptIndexException = Lucene.Net.Index.CorruptIndexException;
-    using DataInput = Lucene.Net.Store.DataInput;
-    using DataOutput = Lucene.Net.Store.DataOutput;
-    using Directory = Lucene.Net.Store.Directory;
-    using FieldInfo = Lucene.Net.Index.FieldInfo;
-    using FieldInfos = Lucene.Net.Index.FieldInfos;
-    using IndexFileNames = Lucene.Net.Index.IndexFileNames;
-    using IndexInput = Lucene.Net.Store.IndexInput;
-    using IOContext = Lucene.Net.Store.IOContext;
-    using IOUtils = Lucene.Net.Util.IOUtils;
-    using PackedInt32s = Lucene.Net.Util.Packed.PackedInt32s;
-    using SegmentInfo = Lucene.Net.Index.SegmentInfo;
-    using StoredFieldVisitor = Lucene.Net.Index.StoredFieldVisitor;
-
-    /// <summary>
-    /// <seealso cref="StoredFieldsReader"/> impl for <seealso cref="CompressingStoredFieldsFormat"/>.
-    /// @lucene.experimental
-    /// </summary>
-    public sealed class CompressingStoredFieldsReader : StoredFieldsReader
-    {
-        // Do not reuse the decompression buffer when there is more than 32kb to decompress
-        private static readonly int BUFFER_REUSE_THRESHOLD = 1 << 15;
-
-        private readonly int version;
-        private readonly FieldInfos fieldInfos;
-        private readonly CompressingStoredFieldsIndexReader indexReader;
-        private readonly long maxPointer;
-        private readonly IndexInput fieldsStream;
-        private readonly int chunkSize;
-        private readonly int packedIntsVersion;
-        private readonly CompressionMode compressionMode;
-        private readonly Decompressor decompressor;
-        private readonly BytesRef bytes;
-        private readonly int numDocs;
-        private bool closed;
-
-        // used by clone
-        private CompressingStoredFieldsReader(CompressingStoredFieldsReader reader)
-        {
-            this.version = reader.version;
-            this.fieldInfos = reader.fieldInfos;
-            this.fieldsStream = (IndexInput)reader.fieldsStream.Clone();
-            this.indexReader = (CompressingStoredFieldsIndexReader)reader.indexReader.Clone();
-            this.maxPointer = reader.maxPointer;
-            this.chunkSize = reader.chunkSize;
-            this.packedIntsVersion = reader.packedIntsVersion;
-            this.compressionMode = reader.compressionMode;
-            this.decompressor = (Decompressor)reader.decompressor.Clone();
-            this.numDocs = reader.numDocs;
-            this.bytes = new BytesRef(reader.bytes.Bytes.Length);
-            this.closed = false;
-        }
-
-        /// <summary>
-        /// Sole constructor. </summary>
-        public CompressingStoredFieldsReader(Directory d, SegmentInfo si, string segmentSuffix, FieldInfos fn, IOContext context, string formatName, CompressionMode compressionMode)
-        {
-            this.compressionMode = compressionMode;
-            string segment = si.Name;
-            bool success = false;
-            fieldInfos = fn;
-            numDocs = si.DocCount;
-            ChecksumIndexInput indexStream = null;
-            try
-            {
-                string indexStreamFN = IndexFileNames.SegmentFileName(segment, segmentSuffix, Lucene40StoredFieldsWriter.FIELDS_INDEX_EXTENSION);
-                string fieldsStreamFN = IndexFileNames.SegmentFileName(segment, segmentSuffix, Lucene40StoredFieldsWriter.FIELDS_EXTENSION);
-                // Load the index into memory
-                indexStream = d.OpenChecksumInput(indexStreamFN, context);
-                string codecNameIdx = formatName + CompressingStoredFieldsWriter.CODEC_SFX_IDX;
-                version = CodecUtil.CheckHeader(indexStream, codecNameIdx, CompressingStoredFieldsWriter.VERSION_START, CompressingStoredFieldsWriter.VERSION_CURRENT);
-                Debug.Assert(CodecUtil.HeaderLength(codecNameIdx) == indexStream.GetFilePointer());
-                indexReader = new CompressingStoredFieldsIndexReader(indexStream, si);
-
-                long maxPointer = -1;
-
-                if (version >= CompressingStoredFieldsWriter.VERSION_CHECKSUM)
-                {
-                    maxPointer = indexStream.ReadVInt64();
-                    CodecUtil.CheckFooter(indexStream);
-                }
-                else
-                {
-#pragma warning disable 612, 618
-                    CodecUtil.CheckEOF(indexStream);
-#pragma warning restore 612, 618
-                }
-                indexStream.Dispose();
-                indexStream = null;
-
-                // Open the data file and read metadata
-                fieldsStream = d.OpenInput(fieldsStreamFN, context);
-                if (version >= CompressingStoredFieldsWriter.VERSION_CHECKSUM)
-                {
-                    if (maxPointer + CodecUtil.FooterLength() != fieldsStream.Length)
-                    {
-                        throw new CorruptIndexException("Invalid fieldsStream maxPointer (file truncated?): maxPointer=" + maxPointer + ", length=" + fieldsStream.Length);
-                    }
-                }
-                else
-                {
-                    maxPointer = fieldsStream.Length;
-                }
-                this.maxPointer = maxPointer;
-                string codecNameDat = formatName + CompressingStoredFieldsWriter.CODEC_SFX_DAT;
-                int fieldsVersion = CodecUtil.CheckHeader(fieldsStream, codecNameDat, CompressingStoredFieldsWriter.VERSION_START, CompressingStoredFieldsWriter.VERSION_CURRENT);
-                if (version != fieldsVersion)
-                {
-                    throw new CorruptIndexException("Version mismatch between stored fields index and data: " + version + " != " + fieldsVersion);
-                }
-                Debug.Assert(CodecUtil.HeaderLength(codecNameDat) == fieldsStream.GetFilePointer());
-
-                if (version >= CompressingStoredFieldsWriter.VERSION_BIG_CHUNKS)
-                {
-                    chunkSize = fieldsStream.ReadVInt32();
-                }
-                else
-                {
-                    chunkSize = -1;
-                }
-                packedIntsVersion = fieldsStream.ReadVInt32();
-                decompressor = compressionMode.NewDecompressor();
-                this.bytes = new BytesRef();
-
-                success = true;
-            }
-            finally
-            {
-                if (!success)
-                {
-                    IOUtils.CloseWhileHandlingException(this, indexStream);
-                }
-            }
-        }
-
-        /// <exception cref="ObjectDisposedException"> if this FieldsReader is closed </exception>
-        private void EnsureOpen()
-        {
-            if (closed)
-            {
-                throw new ObjectDisposedException(this.GetType().GetTypeInfo().FullName, "this FieldsReader is closed");
-            }
-        }
-
-        /// <summary>
-        /// Close the underlying <seealso cref="IndexInput"/>s.
-        /// </summary>
-        protected override void Dispose(bool disposing)
-        {
-            if (!closed)
-            {
-                IOUtils.Close(fieldsStream);
-                closed = true;
-            }
-        }
-
-        private static void ReadField(DataInput @in, StoredFieldVisitor visitor, FieldInfo info, int bits)
-        {
-            switch (bits & CompressingStoredFieldsWriter.TYPE_MASK)
-            {
-                case CompressingStoredFieldsWriter.BYTE_ARR:
-                    int length = @in.ReadVInt32();
-                    var data = new byte[length];
-                    @in.ReadBytes(data, 0, length);
-                    visitor.BinaryField(info, data);
-                    break;
-
-                case CompressingStoredFieldsWriter.STRING:
-                    length = @in.ReadVInt32();
-                    data = new byte[length];
-                    @in.ReadBytes(data, 0, length);
-#pragma warning disable 612, 618
-                    visitor.StringField(info, IOUtils.CHARSET_UTF_8.GetString(data));
-#pragma warning restore 612, 618
-                    break;
-
-                case CompressingStoredFieldsWriter.NUMERIC_INT32:
-                    visitor.Int32Field(info, @in.ReadInt32());
-                    break;
-
-                case CompressingStoredFieldsWriter.NUMERIC_SINGLE:
-                    visitor.SingleField(info, Number.Int32BitsToSingle(@in.ReadInt32()));
-                    break;
-
-                case CompressingStoredFieldsWriter.NUMERIC_INT64:
-                    visitor.Int64Field(info, @in.ReadInt64());
-                    break;
-
-                case CompressingStoredFieldsWriter.NUMERIC_DOUBLE:
-                    visitor.DoubleField(info, BitConverter.Int64BitsToDouble(@in.ReadInt64()));
-                    break;
-
-                default:
-                    throw new InvalidOperationException("Unknown type flag: " + bits.ToString("x"));
-            }
-        }
-
-        private static void SkipField(DataInput @in, int bits)
-        {
-            switch (bits & CompressingStoredFieldsWriter.TYPE_MASK)
-            {
-                case CompressingStoredFieldsWriter.BYTE_ARR:
-                case CompressingStoredFieldsWriter.STRING:
-                    int length = @in.ReadVInt32();
-                    @in.SkipBytes(length);
-                    break;
-
-                case CompressingStoredFieldsWriter.NUMERIC_INT32:
-                case CompressingStoredFieldsWriter.NUMERIC_SINGLE:
-                    @in.ReadInt32();
-                    break;
-
-                case CompressingStoredFieldsWriter.NUMERIC_INT64:
-                case CompressingStoredFieldsWriter.NUMERIC_DOUBLE:
-                    @in.ReadInt64();
-                    break;
-
-                default:
-                    throw new InvalidOperationException("Unknown type flag: " + bits.ToString("x"));
-            }
-        }
-
-        public override void VisitDocument(int docID, StoredFieldVisitor visitor)
-        {
-            fieldsStream.Seek(indexReader.GetStartPointer(docID));
-
-            int docBase = fieldsStream.ReadVInt32();
-            int chunkDocs = fieldsStream.ReadVInt32();
-            if (docID < docBase || docID >= docBase + chunkDocs || docBase + chunkDocs > numDocs)
-            {
-                throw new CorruptIndexException("Corrupted: docID=" + docID + ", docBase=" + docBase + ", chunkDocs=" + chunkDocs + ", numDocs=" + numDocs + " (resource=" + fieldsStream + ")");
-            }
-
-            int numStoredFields, offset, length, totalLength;
-            if (chunkDocs == 1)
-            {
-                numStoredFields = fieldsStream.ReadVInt32();
-                offset = 0;
-                length = fieldsStream.ReadVInt32();
-                totalLength = length;
-            }
-            else
-            {
-                int bitsPerStoredFields = fieldsStream.ReadVInt32();
-                if (bitsPerStoredFields == 0)
-                {
-                    numStoredFields = fieldsStream.ReadVInt32();
-                }
-                else if (bitsPerStoredFields > 31)
-                {
-                    throw new CorruptIndexException("bitsPerStoredFields=" + bitsPerStoredFields + " (resource=" + fieldsStream + ")");
-                }
-                else
-                {
-                    long filePointer = fieldsStream.GetFilePointer();
-                    PackedInt32s.Reader reader = PackedInt32s.GetDirectReaderNoHeader(fieldsStream, PackedInt32s.Format.PACKED, packedIntsVersion, chunkDocs, bitsPerStoredFields);
-                    numStoredFields = (int)(reader.Get(docID - docBase));
-                    fieldsStream.Seek(filePointer + PackedInt32s.Format.PACKED.ByteCount(packedIntsVersion, chunkDocs, bitsPerStoredFields));
-                }
-
-                int bitsPerLength = fieldsStream.ReadVInt32();
-                if (bitsPerLength == 0)
-                {
-                    length = fieldsStream.ReadVInt32();
-                    offset = (docID - docBase) * length;
-                    totalLength = chunkDocs * length;
-                }
-                else if (bitsPerStoredFields > 31)
-                {
-                    throw new CorruptIndexException("bitsPerLength=" + bitsPerLength + " (resource=" + fieldsStream + ")");
-                }
-                else
-                {
-                    PackedInt32s.IReaderIterator it = PackedInt32s.GetReaderIteratorNoHeader(fieldsStream, PackedInt32s.Format.PACKED, packedIntsVersion, chunkDocs, bitsPerLength, 1);
-                    int off = 0;
-                    for (int i = 0; i < docID - docBase; ++i)
-                    {
-                        off += (int)it.Next();
-                    }
-                    offset = off;
-                    length = (int)it.Next();
-                    off += length;
-                    for (int i = docID - docBase + 1; i < chunkDocs; ++i)
-                    {
-                        off += (int)it.Next();
-                    }
-                    totalLength = off;
-                }
-            }
-
-            if ((length == 0) != (numStoredFields == 0))
-            {
-                throw new CorruptIndexException("length=" + length + ", numStoredFields=" + numStoredFields + " (resource=" + fieldsStream + ")");
-            }
-            if (numStoredFields == 0)
-            {
-                // nothing to do
-                return;
-            }
-
-            DataInput documentInput;
-            if (version >= CompressingStoredFieldsWriter.VERSION_BIG_CHUNKS && totalLength >= 2 * chunkSize)
-            {
-                Debug.Assert(chunkSize > 0);
-                Debug.Assert(offset < chunkSize);
-
-                decompressor.Decompress(fieldsStream, chunkSize, offset, Math.Min(length, chunkSize - offset), bytes);
-                documentInput = new DataInputAnonymousInnerClassHelper(this, offset, length);
-            }
-            else
-            {
-                BytesRef bytes = totalLength <= BUFFER_REUSE_THRESHOLD ? this.bytes : new BytesRef();
-                decompressor.Decompress(fieldsStream, totalLength, offset, length, bytes);
-                Debug.Assert(bytes.Length == length);
-                documentInput = new ByteArrayDataInput(bytes.Bytes, bytes.Offset, bytes.Length);
-            }
-
-            for (int fieldIDX = 0; fieldIDX < numStoredFields; fieldIDX++)
-            {
-                long infoAndBits = documentInput.ReadVInt64();
-                int fieldNumber = (int)((long)((ulong)infoAndBits >> CompressingStoredFieldsWriter.TYPE_BITS));
-                FieldInfo fieldInfo = fieldInfos.FieldInfo(fieldNumber);
-
-                int bits = (int)(infoAndBits & CompressingStoredFieldsWriter.TYPE_MASK);
-                Debug.Assert(bits <= CompressingStoredFieldsWriter.NUMERIC_DOUBLE, "bits=" + bits.ToString("x"));
-
-                switch (visitor.NeedsField(fieldInfo))
-                {
-                    case StoredFieldVisitor.Status.YES:
-                        ReadField(documentInput, visitor, fieldInfo, bits);
-                        break;
-
-                    case StoredFieldVisitor.Status.NO:
-                        SkipField(documentInput, bits);
-                        break;
-
-                    case StoredFieldVisitor.Status.STOP:
-                        return;
-                }
-            }
-        }
-
-        private class DataInputAnonymousInnerClassHelper : DataInput
-        {
-            private readonly CompressingStoredFieldsReader outerInstance;
-
-            private int offset;
-            private int length;
-
-            public DataInputAnonymousInnerClassHelper(CompressingStoredFieldsReader outerInstance, int offset, int length)
-            {
-                this.outerInstance = outerInstance;
-                this.offset = offset;
-                this.length = length;
-                decompressed = outerInstance.bytes.Length;
-            }
-
-            internal int decompressed;
-
-            internal virtual void FillBuffer()
-            {
-                Debug.Assert(decompressed <= length);
-                if (decompressed == length)
-                {
-                    throw new Exception();
-                }
-                int toDecompress = Math.Min(length - decompressed, outerInstance.chunkSize);
-                outerInstance.decompressor.Decompress(outerInstance.fieldsStream, toDecompress, 0, toDecompress, outerInstance.bytes);
-                decompressed += toDecompress;
-            }
-
-            public override byte ReadByte()
-            {
-                if (outerInstance.bytes.Length == 0)
-                {
-                    FillBuffer();
-                }
-                --outerInstance.bytes.Length;
-                return (byte)outerInstance.bytes.Bytes[outerInstance.bytes.Offset++];
-            }
-
-            public override void ReadBytes(byte[] b, int offset, int len)
-            {
-                while (len > outerInstance.bytes.Length)
-                {
-                    Array.Copy(outerInstance.bytes.Bytes, outerInstance.bytes.Offset, b, offset, outerInstance.bytes.Length);
-                    len -= outerInstance.bytes.Length;
-                    offset += outerInstance.bytes.Length;
-                    FillBuffer();
-                }
-                Array.Copy(outerInstance.bytes.Bytes, outerInstance.bytes.Offset, b, offset, len);
-                outerInstance.bytes.Offset += len;
-                outerInstance.bytes.Length -= len;
-            }
-        }
-
-        public override object Clone()
-        {
-            EnsureOpen();
-            return new CompressingStoredFieldsReader(this);
-        }
-
-        internal int Version
-        {
-            get
-            {
-                return version;
-            }
-        }
-
-        internal CompressionMode CompressionMode
-        {
-            get
-            {
-                return compressionMode;
-            }
-        }
-
-        internal int ChunkSize
-        {
-            get
-            {
-                return chunkSize;
-            }
-        }
-
-        internal ChunkIterator GetChunkIterator(int startDocID)
-        {
-            EnsureOpen();
-            return new ChunkIterator(this, startDocID);
-        }
-
-        internal sealed class ChunkIterator
-        {
-            private readonly CompressingStoredFieldsReader outerInstance;
-
-            internal readonly ChecksumIndexInput fieldsStream;
-            internal readonly BytesRef spare;
-            internal readonly BytesRef bytes;
-            internal int docBase;
-            internal int chunkDocs;
-            internal int[] numStoredFields;
-            internal int[] lengths;
-
-            internal ChunkIterator(CompressingStoredFieldsReader outerInstance, int startDocId)
-            {
-                this.outerInstance = outerInstance;
-                this.docBase = -1;
-                bytes = new BytesRef();
-                spare = new BytesRef();
-                numStoredFields = new int[1];
-                lengths = new int[1];
-
-                IndexInput @in = outerInstance.fieldsStream;
-                @in.Seek(0);
-                fieldsStream = new BufferedChecksumIndexInput(@in);
-                fieldsStream.Seek(outerInstance.indexReader.GetStartPointer(startDocId));
-            }
-
-            /// <summary>
-            /// Return the decompressed size of the chunk
-            /// </summary>
-            internal int ChunkSize()
-            {
-                int sum = 0;
-                for (int i = 0; i < chunkDocs; ++i)
-                {
-                    sum += lengths[i];
-                }
-                return sum;
-            }
-
-            /// <summary>
-            /// Go to the chunk containing the provided doc ID.
-            /// </summary>
-            internal void Next(int doc)
-            {
-                Debug.Assert(doc >= this.docBase + this.chunkDocs, doc + " " + this.docBase + " " + this.chunkDocs);
-                fieldsStream.Seek(outerInstance.indexReader.GetStartPointer(doc));
-
-                int docBase = fieldsStream.ReadVInt32();
-                int chunkDocs = fieldsStream.ReadVInt32();
-                if (docBase < this.docBase + this.chunkDocs || docBase + chunkDocs > outerInstance.numDocs)
-                {
-                    throw new CorruptIndexException("Corrupted: current docBase=" + this.docBase + ", current numDocs=" + this.chunkDocs + ", new docBase=" + docBase + ", new numDocs=" + chunkDocs + " (resource=" + fieldsStream + ")");
-                }
-                this.docBase = docBase;
-                this.chunkDocs = chunkDocs;
-
-                if (chunkDocs > numStoredFields.Length)
-                {
-                    int newLength = ArrayUtil.Oversize(chunkDocs, 4);
-                    numStoredFields = new int[newLength];
-                    lengths = new int[newLength];
-                }
-
-                if (chunkDocs == 1)
-                {
-                    numStoredFields[0] = fieldsStream.ReadVInt32();
-                    lengths[0] = fieldsStream.ReadVInt32();
-                }
-                else
-                {
-                    int bitsPerStoredFields = fieldsStream.ReadVInt32();
-                    if (bitsPerStoredFields == 0)
-                    {
-                        Arrays.Fill(numStoredFields, 0, chunkDocs, fieldsStream.ReadVInt32());
-                    }
-                    else if (bitsPerStoredFields > 31)
-                    {
-                        throw new CorruptIndexException("bitsPerStoredFields=" + bitsPerStoredFields + " (resource=" + fieldsStream + ")");
-                    }
-                    else
-                    {
-                        PackedInt32s.IReaderIterator it = PackedInt32s.GetReaderIteratorNoHeader(fieldsStream, PackedInt32s.Format.PACKED, outerInstance.packedIntsVersion, chunkDocs, bitsPerStoredFields, 1);
-                        for (int i = 0; i < chunkDocs; ++i)
-                        {
-                            numStoredFields[i] = (int)it.Next();
-                        }
-                    }
-
-                    int bitsPerLength = fieldsStream.ReadVInt32();
-                    if (bitsPerLength == 0)
-                    {
-                        Arrays.Fill(lengths, 0, chunkDocs, fieldsStream.ReadVInt32());
-                    }
-                    else if (bitsPerLength > 31)
-                    {
-                        throw new CorruptIndexException("bitsPerLength=" + bitsPerLength);
-                    }
-                    else
-                    {
-                        PackedInt32s.IReaderIterator it = PackedInt32s.GetReaderIteratorNoHeader(fieldsStream, PackedInt32s.Format.PACKED, outerInstance.packedIntsVersion, chunkDocs, bitsPerLength, 1);
-                        for (int i = 0; i < chunkDocs; ++i)
-                        {
-                            lengths[i] = (int)it.Next();
-                        }
-                    }
-                }
-            }
-
-            /// <summary>
-            /// Decompress the chunk.
-            /// </summary>
-            internal void Decompress()
-            {
-                // decompress data
-                int chunkSize = ChunkSize();
-                if (outerInstance.version >= CompressingStoredFieldsWriter.VERSION_BIG_CHUNKS && chunkSize >= 2 * outerInstance.chunkSize)
-                {
-                    bytes.Offset = bytes.Length = 0;
-                    for (int decompressed = 0; decompressed < chunkSize; )
-                    {
-                        int toDecompress = Math.Min(chunkSize - decompressed, outerInstance.chunkSize);
-                        outerInstance.decompressor.Decompress(fieldsStream, toDecompress, 0, toDecompress, spare);
-                        bytes.Bytes = ArrayUtil.Grow(bytes.Bytes, bytes.Length + spare.Length);
-                        Array.Copy(spare.Bytes, spare.Offset, bytes.Bytes, bytes.Length, spare.Length);
-                        bytes.Length += spare.Length;
-                        decompressed += toDecompress;
-                    }
-                }
-                else
-                {
-                    outerInstance.decompressor.Decompress(fieldsStream, chunkSize, 0, chunkSize, bytes);
-                }
-                if (bytes.Length != chunkSize)
-                {
-                    throw new CorruptIndexException("Corrupted: expected chunk size = " + ChunkSize() + ", got " + bytes.Length + " (resource=" + fieldsStream + ")");
-                }
-            }
-
-            /// <summary>
-            /// Copy compressed data.
-            /// </summary>
-            internal void CopyCompressedData(DataOutput @out)
-            {
-                Debug.Assert(outerInstance.Version == CompressingStoredFieldsWriter.VERSION_CURRENT);
-                long chunkEnd = docBase + chunkDocs == outerInstance.numDocs ? outerInstance.maxPointer : outerInstance.indexReader.GetStartPointer(docBase + chunkDocs);
-                @out.CopyBytes(fieldsStream, chunkEnd - fieldsStream.GetFilePointer());
-            }
-
-            /// <summary>
-            /// Check integrity of the data. The iterator is not usable after this method has been called.
-            /// </summary>
-            internal void CheckIntegrity()
-            {
-                if (outerInstance.version >= CompressingStoredFieldsWriter.VERSION_CHECKSUM)
-                {
-                    fieldsStream.Seek(fieldsStream.Length - CodecUtil.FooterLength());
-                    CodecUtil.CheckFooter(fieldsStream);
-                }
-            }
-        }
-
-        public override long RamBytesUsed()
-        {
-            return indexReader.RamBytesUsed();
-        }
-
-        public override void CheckIntegrity()
-        {
-            if (version >= CompressingStoredFieldsWriter.VERSION_CHECKSUM)
-            {
-                CodecUtil.ChecksumEntireFile(fieldsStream);
-            }
-        }
-    }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/a5dc68d0/src/Lucene.Net.Core/Codecs/Compressing/CompressingStoredFieldsWriter.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Core/Codecs/Compressing/CompressingStoredFieldsWriter.cs b/src/Lucene.Net.Core/Codecs/Compressing/CompressingStoredFieldsWriter.cs
deleted file mode 100644
index 98336fa..0000000
--- a/src/Lucene.Net.Core/Codecs/Compressing/CompressingStoredFieldsWriter.cs
+++ /dev/null
@@ -1,569 +0,0 @@
-using Lucene.Net.Codecs.Lucene40;
-using Lucene.Net.Index;
-using Lucene.Net.Store;
-using Lucene.Net.Support;
-using Lucene.Net.Util;
-using Lucene.Net.Util.Packed;
-using System;
-using System.Diagnostics;
-using Document = Lucene.Net.Documents.Document;
-
-namespace Lucene.Net.Codecs.Compressing
-{
-    /*
-     * Licensed to the Apache Software Foundation (ASF) under one or more
-     * contributor license agreements.  See the NOTICE file distributed with
-     * this work for additional information regarding copyright ownership.
-     * The ASF licenses this file to You under the Apache License, Version 2.0
-     * (the "License"); you may not use this file except in compliance with
-     * the License.  You may obtain a copy of the License at
-     *
-     *     http://www.apache.org/licenses/LICENSE-2.0
-     *
-     * Unless required by applicable law or agreed to in writing, software
-     * distributed under the License is distributed on an "AS IS" BASIS,
-     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-     * See the License for the specific language governing permissions and
-     * limitations under the License.
-     */
-
-    /// <summary>
-    /// <seealso cref="StoredFieldsWriter"/> impl for <seealso cref="CompressingStoredFieldsFormat"/>.
-    /// @lucene.experimental
-    /// </summary>
-    public sealed class CompressingStoredFieldsWriter : StoredFieldsWriter
-    {
-        // hard limit on the maximum number of documents per chunk
-        internal const int MAX_DOCUMENTS_PER_CHUNK = 128;
-
-        internal const int STRING = 0x00;
-        internal const int BYTE_ARR = 0x01;
-
-        /// <summary>
-        /// NOTE: This was NUMERIC_INT in Lucene
-        /// </summary>
-        internal const int NUMERIC_INT32 = 0x02;
-
-        /// <summary>
-        /// NOTE: This was NUMERIC_FLOAT in Lucene
-        /// </summary>
-        internal const int NUMERIC_SINGLE = 0x03;
-
-        /// <summary>
-        /// NOTE:This was NUMERIC_LONG in Lucene
-        /// </summary>
-        internal const int NUMERIC_INT64 = 0x04;
-        internal const int NUMERIC_DOUBLE = 0x05;
-
-        internal static readonly int TYPE_BITS = PackedInt32s.BitsRequired(NUMERIC_DOUBLE);
-        internal static readonly int TYPE_MASK = (int)PackedInt32s.MaxValue(TYPE_BITS);
-
-        internal const string CODEC_SFX_IDX = "Index";
-        internal const string CODEC_SFX_DAT = "Data";
-        internal const int VERSION_START = 0;
-        internal const int VERSION_BIG_CHUNKS = 1;
-        internal const int VERSION_CHECKSUM = 2;
-        internal const int VERSION_CURRENT = VERSION_CHECKSUM;
-
-        private readonly Directory directory;
-        private readonly string segment;
-        private readonly string segmentSuffix;
-        private CompressingStoredFieldsIndexWriter indexWriter;
-        private IndexOutput fieldsStream;
-
-        private readonly CompressionMode compressionMode;
-        private readonly Compressor compressor;
-        private readonly int chunkSize;
-
-        private readonly GrowableByteArrayDataOutput bufferedDocs;
-        private int[] numStoredFields; // number of stored fields
-        private int[] endOffsets; // end offsets in bufferedDocs
-        private int docBase; // doc ID at the beginning of the chunk
-        private int numBufferedDocs; // docBase + numBufferedDocs == current doc ID
-
-        /// <summary>
-        /// Sole constructor. </summary>
-        public CompressingStoredFieldsWriter(Directory directory, SegmentInfo si, string segmentSuffix, IOContext context, string formatName, CompressionMode compressionMode, int chunkSize)
-        {
-            Debug.Assert(directory != null);
-            this.directory = directory;
-            this.segment = si.Name;
-            this.segmentSuffix = segmentSuffix;
-            this.compressionMode = compressionMode;
-            this.compressor = compressionMode.NewCompressor();
-            this.chunkSize = chunkSize;
-            this.docBase = 0;
-            this.bufferedDocs = new GrowableByteArrayDataOutput(chunkSize);
-            this.numStoredFields = new int[16];
-            this.endOffsets = new int[16];
-            this.numBufferedDocs = 0;
-
-            bool success = false;
-            IndexOutput indexStream = directory.CreateOutput(IndexFileNames.SegmentFileName(segment, segmentSuffix, Lucene40StoredFieldsWriter.FIELDS_INDEX_EXTENSION), context);
-            try
-            {
-                fieldsStream = directory.CreateOutput(IndexFileNames.SegmentFileName(segment, segmentSuffix, Lucene40StoredFieldsWriter.FIELDS_EXTENSION), context);
-
-                string codecNameIdx = formatName + CODEC_SFX_IDX;
-                string codecNameDat = formatName + CODEC_SFX_DAT;
-                CodecUtil.WriteHeader(indexStream, codecNameIdx, VERSION_CURRENT);
-                CodecUtil.WriteHeader(fieldsStream, codecNameDat, VERSION_CURRENT);
-                Debug.Assert(CodecUtil.HeaderLength(codecNameDat) == fieldsStream.GetFilePointer());
-                Debug.Assert(CodecUtil.HeaderLength(codecNameIdx) == indexStream.GetFilePointer());
-
-                indexWriter = new CompressingStoredFieldsIndexWriter(indexStream);
-                indexStream = null;
-
-                fieldsStream.WriteVInt32(chunkSize);
-                fieldsStream.WriteVInt32(PackedInt32s.VERSION_CURRENT);
-
-                success = true;
-            }
-            finally
-            {
-                if (!success)
-                {
-                    IOUtils.CloseWhileHandlingException(indexStream);
-                    Abort();
-                }
-            }
-        }
-
-        protected override void Dispose(bool disposing)
-        {
-            if (disposing)
-            {
-                try
-                {
-                    IOUtils.Close(fieldsStream, indexWriter);
-                }
-                finally
-                {
-                    fieldsStream = null;
-                    indexWriter = null;
-                }
-            }
-        }
-
-        public override void StartDocument(int numStoredFields)
-        {
-            if (numBufferedDocs == this.numStoredFields.Length)
-            {
-                int newLength = ArrayUtil.Oversize(numBufferedDocs + 1, 4);
-                this.numStoredFields = Arrays.CopyOf(this.numStoredFields, newLength);
-                endOffsets = Arrays.CopyOf(endOffsets, newLength);
-            }
-            this.numStoredFields[numBufferedDocs] = numStoredFields;
-            ++numBufferedDocs;
-        }
-
-        public override void FinishDocument()
-        {
-            endOffsets[numBufferedDocs - 1] = bufferedDocs.Length;
-            if (TriggerFlush())
-            {
-                Flush();
-            }
-        }
-
-        /// <summary>
-        /// NOTE: This was saveInts() in Lucene
-        /// </summary>
-        private static void SaveInt32s(int[] values, int length, DataOutput @out)
-        {
-            Debug.Assert(length > 0);
-            if (length == 1)
-            {
-                @out.WriteVInt32(values[0]);
-            }
-            else
-            {
-                bool allEqual = true;
-                for (int i = 1; i < length; ++i)
-                {
-                    if (values[i] != values[0])
-                    {
-                        allEqual = false;
-                        break;
-                    }
-                }
-                if (allEqual)
-                {
-                    @out.WriteVInt32(0);
-                    @out.WriteVInt32(values[0]);
-                }
-                else
-                {
-                    long max = 0;
-                    for (int i = 0; i < length; ++i)
-                    {
-                        max |= (uint)values[i];
-                    }
-                    int bitsRequired = PackedInt32s.BitsRequired(max);
-                    @out.WriteVInt32(bitsRequired);
-                    PackedInt32s.Writer w = PackedInt32s.GetWriterNoHeader(@out, PackedInt32s.Format.PACKED, length, bitsRequired, 1);
-                    for (int i = 0; i < length; ++i)
-                    {
-                        w.Add(values[i]);
-                    }
-                    w.Finish();
-                }
-            }
-        }
-
-        private void WriteHeader(int docBase, int numBufferedDocs, int[] numStoredFields, int[] lengths)
-        {
-            // save docBase and numBufferedDocs
-            fieldsStream.WriteVInt32(docBase);
-            fieldsStream.WriteVInt32(numBufferedDocs);
-
-            // save numStoredFields
-            SaveInt32s(numStoredFields, numBufferedDocs, fieldsStream);
-
-            // save lengths
-            SaveInt32s(lengths, numBufferedDocs, fieldsStream);
-        }
-
-        private bool TriggerFlush()
-        {
-            return bufferedDocs.Length >= chunkSize || numBufferedDocs >= MAX_DOCUMENTS_PER_CHUNK; // chunks of at least chunkSize bytes
-        }
-
-        private void Flush()
-        {
-            indexWriter.WriteIndex(numBufferedDocs, fieldsStream.GetFilePointer());
-
-            // transform end offsets into lengths
-            int[] lengths = endOffsets;
-            for (int i = numBufferedDocs - 1; i > 0; --i)
-            {
-                lengths[i] = endOffsets[i] - endOffsets[i - 1];
-                Debug.Assert(lengths[i] >= 0);
-            }
-            WriteHeader(docBase, numBufferedDocs, numStoredFields, lengths);
-
-            // compress stored fields to fieldsStream
-            if (bufferedDocs.Length >= 2 * chunkSize)
-            {
-                // big chunk, slice it
-                for (int compressed = 0; compressed < bufferedDocs.Length; compressed += chunkSize)
-                {
-                    compressor.Compress(bufferedDocs.Bytes, compressed, Math.Min(chunkSize, bufferedDocs.Length - compressed), fieldsStream);
-                }
-            }
-            else
-            {
-                compressor.Compress(bufferedDocs.Bytes, 0, bufferedDocs.Length, fieldsStream);
-            }
-
-            // reset
-            docBase += numBufferedDocs;
-            numBufferedDocs = 0;
-            bufferedDocs.Length = 0;
-        }
-
-        public override void WriteField(FieldInfo info, IIndexableField field)
-        {
-            int bits = 0;
-            BytesRef bytes;
-            string @string;
-
-            object number = (object)field.GetNumericValue();
-            if (number != null)
-            {
-                if (number is string)
-                {
-                    string numStr = number.ToString();
-                    sbyte dummySbyte;
-                    short dummyShort;
-                    int dummyInt;
-                    long dummyLong;
-                    float dummyFloat;
-                    double dummyDouble;
-                    if (sbyte.TryParse(numStr, out dummySbyte) || short.TryParse(numStr, out dummyShort) || int.TryParse(numStr, out dummyInt))
-                    {
-                        bits = NUMERIC_INT32;
-                    }
-                    else if (long.TryParse(numStr, out dummyLong))
-                    {
-                        bits = NUMERIC_INT64;
-                    }
-                    else if (float.TryParse(numStr, out dummyFloat))
-                    {
-                        bits = NUMERIC_SINGLE;
-                    }
-                    else if (double.TryParse(numStr, out dummyDouble))
-                    {
-                        bits = NUMERIC_DOUBLE;
-                    }
-                    else
-                    {
-                        throw new System.ArgumentException("cannot store numeric type " + number.GetType());
-                    }
-                }
-                else
-                {
-                    if (number is sbyte || number is short || number is int)
-                    {
-                        bits = NUMERIC_INT32;
-                    }
-                    else if (number is long)
-                    {
-                        bits = NUMERIC_INT64;
-                    }
-                    else if (number is float)
-                    {
-                        bits = NUMERIC_SINGLE;
-                    }
-                    else if (number is double)
-                    {
-                        bits = NUMERIC_DOUBLE;
-                    }
-                    else
-                    {
-                        throw new System.ArgumentException("cannot store numeric type " + number.GetType());
-                    }
-                }
-
-                @string = null;
-                bytes = null;
-            }
-            else
-            {
-                bytes = field.GetBinaryValue();
-                if (bytes != null)
-                {
-                    bits = BYTE_ARR;
-                    @string = null;
-                }
-                else
-                {
-                    bits = STRING;
-                    @string = field.GetStringValue();
-                    if (@string == null)
-                    {
-                        throw new System.ArgumentException("field " + field.Name + " is stored but does not have binaryValue, stringValue nor numericValue");
-                    }
-                }
-            }
-
-            long infoAndBits = (((long)info.Number) << TYPE_BITS) | bits;
-            bufferedDocs.WriteVInt64(infoAndBits);
-
-            if (bytes != null)
-            {
-                bufferedDocs.WriteVInt32(bytes.Length);
-                bufferedDocs.WriteBytes(bytes.Bytes, bytes.Offset, bytes.Length);
-            }
-            else if (@string != null)
-            {
-                bufferedDocs.WriteString(field.GetStringValue());
-            }
-            else
-            {
-                if (number is string)
-                {
-                    string numStr = number.ToString();
-                    sbyte dummySbyte;
-                    short dummyShort;
-                    int dummyInt;
-                    long dummyLong;
-                    float dummyFloat;
-                    double dummyDouble;
-                    if (sbyte.TryParse(numStr, out dummySbyte) || short.TryParse(numStr, out dummyShort) ||
-                        int.TryParse(numStr, out dummyInt))
-                    {
-                        bits = NUMERIC_INT32;
-                    }
-                    else if (long.TryParse(numStr, out dummyLong))
-                    {
-                        bits = NUMERIC_INT64;
-                    }
-                    else if (float.TryParse(numStr, out dummyFloat))
-                    {
-                        bits = NUMERIC_SINGLE;
-                    }
-                    else if (double.TryParse(numStr, out dummyDouble))
-                    {
-                        bits = NUMERIC_DOUBLE;
-                    }
-                    else
-                    {
-                        throw new System.ArgumentException("cannot store numeric type " + number.GetType());
-                    }
-                }
-                else
-                {
-                    if (number is sbyte || number is short || number is int)
-                    {
-                        bufferedDocs.WriteInt32((int)number);
-                    }
-                    else if (number is long)
-                    {
-                        bufferedDocs.WriteInt64((long)number);
-                    }
-                    else if (number is float)
-                    {
-                        bufferedDocs.WriteInt32(Number.SingleToInt32Bits((float)number));
-                    }
-                    else if (number is double)
-                    {
-                        bufferedDocs.WriteInt64(BitConverter.DoubleToInt64Bits((double)number));
-                    }
-                    else
-                    {
-                        throw new Exception("Cannot get here");
-                    }
-                }
-            }
-        }
-
-        public override void Abort()
-        {
-            IOUtils.CloseWhileHandlingException(this);
-            IOUtils.DeleteFilesIgnoringExceptions(directory, IndexFileNames.SegmentFileName(segment, segmentSuffix, Lucene40StoredFieldsWriter.FIELDS_EXTENSION), IndexFileNames.SegmentFileName(segment, segmentSuffix, Lucene40StoredFieldsWriter.FIELDS_INDEX_EXTENSION));
-        }
-
-        public override void Finish(FieldInfos fis, int numDocs)
-        {
-            if (numBufferedDocs > 0)
-            {
-                Flush();
-            }
-            else
-            {
-                Debug.Assert(bufferedDocs.Length == 0);
-            }
-            if (docBase != numDocs)
-            {
-                throw new Exception("Wrote " + docBase + " docs, finish called with numDocs=" + numDocs);
-            }
-            indexWriter.Finish(numDocs, fieldsStream.GetFilePointer());
-            CodecUtil.WriteFooter(fieldsStream);
-            Debug.Assert(bufferedDocs.Length == 0);
-        }
-
-        public override int Merge(MergeState mergeState)
-        {
-            int docCount = 0;
-            int idx = 0;
-
-            foreach (AtomicReader reader in mergeState.Readers)
-            {
-                SegmentReader matchingSegmentReader = mergeState.MatchingSegmentReaders[idx++];
-                CompressingStoredFieldsReader matchingFieldsReader = null;
-                if (matchingSegmentReader != null)
-                {
-                    StoredFieldsReader fieldsReader = matchingSegmentReader.FieldsReader;
-                    // we can only bulk-copy if the matching reader is also a CompressingStoredFieldsReader
-                    if (fieldsReader != null && fieldsReader is CompressingStoredFieldsReader)
-                    {
-                        matchingFieldsReader = (CompressingStoredFieldsReader)fieldsReader;
-                    }
-                }
-
-                int maxDoc = reader.MaxDoc;
-                IBits liveDocs = reader.LiveDocs;
-
-                if (matchingFieldsReader == null || matchingFieldsReader.Version != VERSION_CURRENT || matchingFieldsReader.CompressionMode != compressionMode || matchingFieldsReader.ChunkSize != chunkSize) // the way data is decompressed depends on the chunk size -  means reader version is not the same as the writer version
-                {
-                    // naive merge...
-                    for (int i = NextLiveDoc(0, liveDocs, maxDoc); i < maxDoc; i = NextLiveDoc(i + 1, liveDocs, maxDoc))
-                    {
-                        Document doc = reader.Document(i);
-                        AddDocument(doc, mergeState.FieldInfos);
-                        ++docCount;
-                        mergeState.CheckAbort.Work(300);
-                    }
-                }
-                else
-                {
-                    int docID = NextLiveDoc(0, liveDocs, maxDoc);
-                    if (docID < maxDoc)
-                    {
-                        // not all docs were deleted
-                        CompressingStoredFieldsReader.ChunkIterator it = matchingFieldsReader.GetChunkIterator(docID);
-                        int[] startOffsets = new int[0];
-                        do
-                        {
-                            // go to the next chunk that contains docID
-                            it.Next(docID);
-                            // transform lengths into offsets
-                            if (startOffsets.Length < it.chunkDocs)
-                            {
-                                startOffsets = new int[ArrayUtil.Oversize(it.chunkDocs, 4)];
-                            }
-                            for (int i = 1; i < it.chunkDocs; ++i)
-                            {
-                                startOffsets[i] = startOffsets[i - 1] + it.lengths[i - 1];
-                            }
-
-                            if (numBufferedDocs == 0 && startOffsets[it.chunkDocs - 1] < chunkSize && startOffsets[it.chunkDocs - 1] + it.lengths[it.chunkDocs - 1] >= chunkSize && NextDeletedDoc(it.docBase, liveDocs, it.docBase + it.chunkDocs) == it.docBase + it.chunkDocs) // no deletion in the chunk -  chunk is large enough -  chunk is small enough -  starting a new chunk
-                            {
-                                Debug.Assert(docID == it.docBase);
-
-                                // no need to decompress, just copy data
-                                indexWriter.WriteIndex(it.chunkDocs, fieldsStream.GetFilePointer());
-                                WriteHeader(this.docBase, it.chunkDocs, it.numStoredFields, it.lengths);
-                                it.CopyCompressedData(fieldsStream);
-                                this.docBase += it.chunkDocs;
-                                docID = NextLiveDoc(it.docBase + it.chunkDocs, liveDocs, maxDoc);
-                                docCount += it.chunkDocs;
-                                mergeState.CheckAbort.Work(300 * it.chunkDocs);
-                            }
-                            else
-                            {
-                                // decompress
-                                it.Decompress();
-                                if (startOffsets[it.chunkDocs - 1] + it.lengths[it.chunkDocs - 1] != it.bytes.Length)
-                                {
-                                    throw new CorruptIndexException("Corrupted: expected chunk size=" + startOffsets[it.chunkDocs - 1] + it.lengths[it.chunkDocs - 1] + ", got " + it.bytes.Length);
-                                }
-                                // copy non-deleted docs
-                                for (; docID < it.docBase + it.chunkDocs; docID = NextLiveDoc(docID + 1, liveDocs, maxDoc))
-                                {
-                                    int diff = docID - it.docBase;
-                                    StartDocument(it.numStoredFields[diff]);
-                                    bufferedDocs.WriteBytes(it.bytes.Bytes, it.bytes.Offset + startOffsets[diff], it.lengths[diff]);
-                                    FinishDocument();
-                                    ++docCount;
-                                    mergeState.CheckAbort.Work(300);
-                                }
-                            }
-                        } while (docID < maxDoc);
-
-                        it.CheckIntegrity();
-                    }
-                }
-            }
-            Finish(mergeState.FieldInfos, docCount);
-            return docCount;
-        }
-
-        private static int NextLiveDoc(int doc, IBits liveDocs, int maxDoc)
-        {
-            if (liveDocs == null)
-            {
-                return doc;
-            }
-            while (doc < maxDoc && !liveDocs.Get(doc))
-            {
-                ++doc;
-            }
-            return doc;
-        }
-
-        private static int NextDeletedDoc(int doc, IBits liveDocs, int maxDoc)
-        {
-            if (liveDocs == null)
-            {
-                return maxDoc;
-            }
-            while (doc < maxDoc && liveDocs.Get(doc))
-            {
-                ++doc;
-            }
-            return doc;
-        }
-    }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/a5dc68d0/src/Lucene.Net.Core/Codecs/Compressing/CompressingTermVectorsFormat.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Core/Codecs/Compressing/CompressingTermVectorsFormat.cs b/src/Lucene.Net.Core/Codecs/Compressing/CompressingTermVectorsFormat.cs
deleted file mode 100644
index 8952cc5..0000000
--- a/src/Lucene.Net.Core/Codecs/Compressing/CompressingTermVectorsFormat.cs
+++ /dev/null
@@ -1,90 +0,0 @@
-using Directory = Lucene.Net.Store.Directory;
-
-namespace Lucene.Net.Codecs.Compressing
-{
-    /*
-     * Licensed to the Apache Software Foundation (ASF) under one or more
-     * contributor license agreements.  See the NOTICE file distributed with
-     * this work for additional information regarding copyright ownership.
-     * The ASF licenses this file to You under the Apache License, Version 2.0
-     * (the "License"); you may not use this file except in compliance with
-     * the License.  You may obtain a copy of the License at
-     *
-     *     http://www.apache.org/licenses/LICENSE-2.0
-     *
-     * Unless required by applicable law or agreed to in writing, software
-     * distributed under the License is distributed on an "AS IS" BASIS,
-     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-     * See the License for the specific language governing permissions and
-     * limitations under the License.
-     */
-
-    using FieldInfos = Lucene.Net.Index.FieldInfos;
-    using IOContext = Lucene.Net.Store.IOContext;
-    using SegmentInfo = Lucene.Net.Index.SegmentInfo;
-
-    /// <summary>
-    /// A <seealso cref="TermVectorsFormat"/> that compresses chunks of documents together in
-    /// order to improve the compression ratio.
-    /// @lucene.experimental
-    /// </summary>
-    public class CompressingTermVectorsFormat : TermVectorsFormat
-    {
-        private readonly string formatName;
-        private readonly string segmentSuffix;
-        private readonly CompressionMode compressionMode;
-        private readonly int chunkSize;
-
-        /// <summary>
-        /// Create a new <seealso cref="CompressingTermVectorsFormat"/>.
-        /// <p>
-        /// <code>formatName</code> is the name of the format. this name will be used
-        /// in the file formats to perform
-        /// <seealso cref="CodecUtil#checkHeader(Lucene.Net.Store.DataInput, String, int, int) codec header checks"/>.
-        /// <p>
-        /// The <code>compressionMode</code> parameter allows you to choose between
-        /// compression algorithms that have various compression and decompression
-        /// speeds so that you can pick the one that best fits your indexing and
-        /// searching throughput. You should never instantiate two
-        /// <seealso cref="CompressingTermVectorsFormat"/>s that have the same name but
-        /// different <seealso cref="compressionMode"/>s.
-        /// <p>
-        /// <code>chunkSize</code> is the minimum byte size of a chunk of documents.
-        /// Higher values of <code>chunkSize</code> should improve the compression
-        /// ratio but will require more memory at indexing time and might make document
-        /// loading a little slower (depending on the size of your OS cache compared
-        /// to the size of your index).
-        /// </summary>
-        /// <param name="formatName"> the name of the <seealso cref="StoredFieldsFormat"/> </param>
-        /// <param name="segmentSuffix"> a suffix to append to files created by this format </param>
-        /// <param name="compressionMode"> the <seealso cref="compressionMode"/> to use </param>
-        /// <param name="chunkSize"> the minimum number of bytes of a single chunk of stored documents </param>
-        /// <seealso cref= compressionMode </seealso>
-        public CompressingTermVectorsFormat(string formatName, string segmentSuffix, CompressionMode compressionMode, int chunkSize)
-        {
-            this.formatName = formatName;
-            this.segmentSuffix = segmentSuffix;
-            this.compressionMode = compressionMode;
-            if (chunkSize < 1)
-            {
-                throw new System.ArgumentException("chunkSize must be >= 1");
-            }
-            this.chunkSize = chunkSize;
-        }
-
-        public override sealed TermVectorsReader VectorsReader(Directory directory, SegmentInfo segmentInfo, FieldInfos fieldInfos, IOContext context)
-        {
-            return new CompressingTermVectorsReader(directory, segmentInfo, segmentSuffix, fieldInfos, context, formatName, compressionMode);
-        }
-
-        public override sealed TermVectorsWriter VectorsWriter(Directory directory, SegmentInfo segmentInfo, IOContext context)
-        {
-            return new CompressingTermVectorsWriter(directory, segmentInfo, segmentSuffix, context, formatName, compressionMode, chunkSize);
-        }
-
-        public override string ToString()
-        {
-            return this.GetType().Name + "(compressionMode=" + compressionMode + ", chunkSize=" + chunkSize + ")";
-        }
-    }
-}
\ No newline at end of file


Mime
View raw message