lucenenet-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From pnas...@apache.org
Subject [04/10] Lucene.Net.Codes/Sep fully ported, work done on SimpleText and Memory as well
Date Sun, 28 Sep 2014 08:50:08 GMT
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/d852d5b0/src/Lucene.Net.Codecs/Memory/MemoryDocValuesProducer.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Codecs/Memory/MemoryDocValuesProducer.cs b/src/Lucene.Net.Codecs/Memory/MemoryDocValuesProducer.cs
index 015a1af..5ab0b09 100644
--- a/src/Lucene.Net.Codecs/Memory/MemoryDocValuesProducer.cs
+++ b/src/Lucene.Net.Codecs/Memory/MemoryDocValuesProducer.cs
@@ -1,658 +1,995 @@
-package codecs.memory;
-
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import java.io.IOException;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.concurrent.atomic.AtomicLong;
-
-import codecs.CodecUtil;
-import codecs.DocValuesProducer;
-import index.BinaryDocValues;
-import index.CorruptIndexException;
-import index.DocValues;
-import index.DocsAndPositionsEnum;
-import index.DocsEnum;
-import index.FieldInfo;
-import index.FieldInfos;
-import index.IndexFileNames;
-import index.NumericDocValues;
-import index.SegmentReadState;
-import index.SortedDocValues;
-import index.SortedSetDocValues;
-import index.TermsEnum;
-import store.ByteArrayDataInput;
-import store.ChecksumIndexInput;
-import store.IndexInput;
-import util.Bits;
-import util.BytesRef;
-import util.FixedBitSet;
-import util.IOUtils;
-import util.IntsRef;
-import util.PagedBytes;
-import util.RamUsageEstimator;
-import util.fst.BytesRefFSTEnum;
-import util.fst.BytesRefFSTEnum.InputOutput;
-import util.fst.FST;
-import util.fst.FST.Arc;
-import util.fst.FST.BytesReader;
-import util.fst.PositiveIntOutputs;
-import util.fst.Util;
-import util.packed.BlockPackedReader;
-import util.packed.MonotonicBlockPackedReader;
-import util.packed.PackedInts;
-
-/**
- * Reader for {@link MemoryDocValuesFormat}
- */
-class MemoryDocValuesProducer extends DocValuesProducer {
-  // metadata maps (just file pointers and minimal stuff)
-  private final Map<Integer,NumericEntry> numerics;
-  private final Map<Integer,BinaryEntry> binaries;
-  private final Map<Integer,FSTEntry> fsts;
-  private final IndexInput data;
-  
-  // ram instances we have already loaded
-  private final Map<Integer,NumericDocValues> numericInstances = 
-      new HashMap<>();
-  private final Map<Integer,BinaryDocValues> binaryInstances =
-      new HashMap<>();
-  private final Map<Integer,FST<Long>> fstInstances =
-      new HashMap<>();
-  private final Map<Integer,Bits> docsWithFieldInstances = new HashMap<>();
-  
-  private final int maxDoc;
-  private final AtomicLong ramBytesUsed;
-  private final int version;
-  
-  static final byte NUMBER = 0;
-  static final byte BYTES = 1;
-  static final byte FST = 2;
-
-  static final int BLOCK_SIZE = 4096;
-  
-  static final byte DELTA_COMPRESSED = 0;
-  static final byte TABLE_COMPRESSED = 1;
-  static final byte UNCOMPRESSED = 2;
-  static final byte GCD_COMPRESSED = 3;
-  
-  static final int VERSION_START = 0;
-  static final int VERSION_GCD_COMPRESSION = 1;
-  static final int VERSION_CHECKSUM = 2;
-  static final int VERSION_CURRENT = VERSION_CHECKSUM;
-    
-  MemoryDocValuesProducer(SegmentReadState state, String dataCodec, String dataExtension, String metaCodec, String metaExtension)  {
-    maxDoc = state.segmentInfo.getDocCount();
-    String metaName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, metaExtension);
-    // read in the entries from the metadata file.
-    ChecksumIndexInput in = state.directory.openChecksumInput(metaName, state.context);
-    bool success = false;
-    try {
-      version = CodecUtil.checkHeader(in, metaCodec, 
-                                      VERSION_START,
-                                      VERSION_CURRENT);
-      numerics = new HashMap<>();
-      binaries = new HashMap<>();
-      fsts = new HashMap<>();
-      readFields(in, state.fieldInfos);
-      if (version >= VERSION_CHECKSUM) {
-        CodecUtil.checkFooter(in);
-      } else {
-        CodecUtil.checkEOF(in);
-      }
-      ramBytesUsed = new AtomicLong(RamUsageEstimator.shallowSizeOfInstance(getClass()));
-      success = true;
-    } finally {
-      if (success) {
-        IOUtils.close(in);
-      } else {
-        IOUtils.closeWhileHandlingException(in);
-      }
-    }
-
-    success = false;
-    try {
-      String dataName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, dataExtension);
-      data = state.directory.openInput(dataName, state.context);
-      final int version2 = CodecUtil.checkHeader(data, dataCodec, 
-                                                 VERSION_START,
-                                                 VERSION_CURRENT);
-      if (version != version2) {
-        throw new CorruptIndexException("Format versions mismatch");
-      }
-
-      success = true;
-    } finally {
-      if (!success) {
-        IOUtils.closeWhileHandlingException(this.data);
-      }
-    }
-  }
-  
-  private void readFields(IndexInput meta, FieldInfos infos)  {
-    int fieldNumber = meta.readVInt();
-    while (fieldNumber != -1) {
-      int fieldType = meta.readByte();
-      if (fieldType == NUMBER) {
-        NumericEntry entry = new NumericEntry();
-        entry.offset = meta.readLong();
-        entry.missingOffset = meta.readLong();
-        if (entry.missingOffset != -1) {
-          entry.missingBytes = meta.readLong();
-        } else {
-          entry.missingBytes = 0;
-        }
-        entry.format = meta.readByte();
-        switch(entry.format) {
-          case DELTA_COMPRESSED:
-          case TABLE_COMPRESSED:
-          case GCD_COMPRESSED:
-          case UNCOMPRESSED:
-               break;
-          default:
-               throw new CorruptIndexException("Unknown format: " + entry.format + ", input=" + meta);
-        }
-        if (entry.format != UNCOMPRESSED) {
-          entry.packedIntsVersion = meta.readVInt();
-        }
-        numerics.put(fieldNumber, entry);
-      } else if (fieldType == BYTES) {
-        BinaryEntry entry = new BinaryEntry();
-        entry.offset = meta.readLong();
-        entry.numBytes = meta.readLong();
-        entry.missingOffset = meta.readLong();
-        if (entry.missingOffset != -1) {
-          entry.missingBytes = meta.readLong();
-        } else {
-          entry.missingBytes = 0;
-        }
-        entry.minLength = meta.readVInt();
-        entry.maxLength = meta.readVInt();
-        if (entry.minLength != entry.maxLength) {
-          entry.packedIntsVersion = meta.readVInt();
-          entry.blockSize = meta.readVInt();
-        }
-        binaries.put(fieldNumber, entry);
-      } else if (fieldType == FST) {
-        FSTEntry entry = new FSTEntry();
-        entry.offset = meta.readLong();
-        entry.numOrds = meta.readVLong();
-        fsts.put(fieldNumber, entry);
-      } else {
-        throw new CorruptIndexException("invalid entry type: " + fieldType + ", input=" + meta);
-      }
-      fieldNumber = meta.readVInt();
-    }
-  }
-
-  @Override
-  public synchronized NumericDocValues getNumeric(FieldInfo field)  {
-    NumericDocValues instance = numericInstances.get(field.number);
-    if (instance == null) {
-      instance = loadNumeric(field);
-      numericInstances.put(field.number, instance);
-    }
-    return instance;
-  }
-  
-  @Override
-  public long ramBytesUsed() {
-    return ramBytesUsed.get();
-  }
-  
-  @Override
-  public void checkIntegrity()  {
-    if (version >= VERSION_CHECKSUM) {
-      CodecUtil.checksumEntireFile(data);
-    }
-  }
-  
-  private NumericDocValues loadNumeric(FieldInfo field)  {
-    NumericEntry entry = numerics.get(field.number);
-    data.seek(entry.offset + entry.missingBytes);
-    switch (entry.format) {
-      case TABLE_COMPRESSED:
-        int size = data.readVInt();
-        if (size > 256) {
-          throw new CorruptIndexException("TABLE_COMPRESSED cannot have more than 256 distinct values, input=" + data);
-        }
-        final long decode[] = new long[size];
-        for (int i = 0; i < decode.length; i++) {
-          decode[i] = data.readLong();
-        }
-        final int formatID = data.readVInt();
-        final int bitsPerValue = data.readVInt();
-        final PackedInts.Reader ordsReader = PackedInts.getReaderNoHeader(data, PackedInts.Format.byId(formatID), entry.packedIntsVersion, maxDoc, bitsPerValue);
-        ramBytesUsed.addAndGet(RamUsageEstimator.sizeOf(decode) + ordsReader.ramBytesUsed());
-        return new NumericDocValues() {
-          @Override
-          public long get(int docID) {
-            return decode[(int)ordsReader.get(docID)];
-          }
-        };
-      case DELTA_COMPRESSED:
-        final int blockSize = data.readVInt();
-        final BlockPackedReader reader = new BlockPackedReader(data, entry.packedIntsVersion, blockSize, maxDoc, false);
-        ramBytesUsed.addAndGet(reader.ramBytesUsed());
-        return reader;
-      case UNCOMPRESSED:
-        final byte bytes[] = new byte[maxDoc];
-        data.readBytes(bytes, 0, bytes.length);
-        ramBytesUsed.addAndGet(RamUsageEstimator.sizeOf(bytes));
-        return new NumericDocValues() {
-          @Override
-          public long get(int docID) {
-            return bytes[docID];
-          }
-        };
-      case GCD_COMPRESSED:
-        final long min = data.readLong();
-        final long mult = data.readLong();
-        final int quotientBlockSize = data.readVInt();
-        final BlockPackedReader quotientReader = new BlockPackedReader(data, entry.packedIntsVersion, quotientBlockSize, maxDoc, false);
-        ramBytesUsed.addAndGet(quotientReader.ramBytesUsed());
-        return new NumericDocValues() {
-          @Override
-          public long get(int docID) {
-            return min + mult * quotientReader.get(docID);
-          }
-        };
-      default:
-        throw new Debug.Assert(ionError();
-    }
-  }
-
-  @Override
-  public synchronized BinaryDocValues getBinary(FieldInfo field)  {
-    BinaryDocValues instance = binaryInstances.get(field.number);
-    if (instance == null) {
-      instance = loadBinary(field);
-      binaryInstances.put(field.number, instance);
-    }
-    return instance;
-  }
-  
-  private BinaryDocValues loadBinary(FieldInfo field)  {
-    BinaryEntry entry = binaries.get(field.number);
-    data.seek(entry.offset);
-    PagedBytes bytes = new PagedBytes(16);
-    bytes.copy(data, entry.numBytes);
-    final PagedBytes.Reader bytesReader = bytes.freeze(true);
-    if (entry.minLength == entry.maxLength) {
-      final int fixedLength = entry.minLength;
-      ramBytesUsed.addAndGet(bytes.ramBytesUsed());
-      return new BinaryDocValues() {
-        @Override
-        public void get(int docID, BytesRef result) {
-          bytesReader.fillSlice(result, fixedLength * (long)docID, fixedLength);
-        }
-      };
-    } else {
-      data.seek(data.getFilePointer() + entry.missingBytes);
-      final MonotonicBlockPackedReader addresses = new MonotonicBlockPackedReader(data, entry.packedIntsVersion, entry.blockSize, maxDoc, false);
-      ramBytesUsed.addAndGet(bytes.ramBytesUsed() + addresses.ramBytesUsed());
-      return new BinaryDocValues() {
-        @Override
-        public void get(int docID, BytesRef result) {
-          long startAddress = docID == 0 ? 0 : addresses.get(docID-1);
-          long endAddress = addresses.get(docID); 
-          bytesReader.fillSlice(result, startAddress, (int) (endAddress - startAddress));
-        }
-      };
-    }
-  }
-  
-  @Override
-  public SortedDocValues getSorted(FieldInfo field)  {
-    final FSTEntry entry = fsts.get(field.number);
-    if (entry.numOrds == 0) {
-      return DocValues.EMPTY_SORTED;
-    }
-    FST<Long> instance;
-    synchronized(this) {
-      instance = fstInstances.get(field.number);
-      if (instance == null) {
-        data.seek(entry.offset);
-        instance = new FST<>(data, PositiveIntOutputs.getSingleton());
-        ramBytesUsed.addAndGet(instance.sizeInBytes());
-        fstInstances.put(field.number, instance);
-      }
-    }
-    final NumericDocValues docToOrd = getNumeric(field);
-    final FST<Long> fst = instance;
-    
-    // per-thread resources
-    final BytesReader in = fst.getBytesReader();
-    final Arc<Long> firstArc = new Arc<>();
-    final Arc<Long> scratchArc = new Arc<>();
-    final IntsRef scratchInts = new IntsRef();
-    final BytesRefFSTEnum<Long> fstEnum = new BytesRefFSTEnum<>(fst);
-    
-    return new SortedDocValues() {
-      @Override
-      public int getOrd(int docID) {
-        return (int) docToOrd.get(docID);
-      }
-
-      @Override
-      public void lookupOrd(int ord, BytesRef result) {
-        try {
-          in.setPosition(0);
-          fst.getFirstArc(firstArc);
-          IntsRef output = Util.getByOutput(fst, ord, in, firstArc, scratchArc, scratchInts);
-          result.bytes = new byte[output.length];
-          result.offset = 0;
-          result.length = 0;
-          Util.toBytesRef(output, result);
-        } catch (IOException bogus) {
-          throw new RuntimeException(bogus);
-        }
-      }
-
-      @Override
-      public int lookupTerm(BytesRef key) {
-        try {
-          InputOutput<Long> o = fstEnum.seekCeil(key);
-          if (o == null) {
-            return -getValueCount()-1;
-          } else if (o.input.equals(key)) {
-            return o.output.intValue();
-          } else {
-            return (int) -o.output-1;
-          }
-        } catch (IOException bogus) {
-          throw new RuntimeException(bogus);
-        }
-      }
-
-      @Override
-      public int getValueCount() {
-        return (int)entry.numOrds;
-      }
-
-      @Override
-      public TermsEnum termsEnum() {
-        return new FSTTermsEnum(fst);
-      }
-    };
-  }
-  
-  @Override
-  public SortedSetDocValues getSortedSet(FieldInfo field)  {
-    final FSTEntry entry = fsts.get(field.number);
-    if (entry.numOrds == 0) {
-      return DocValues.EMPTY_SORTED_SET; // empty FST!
-    }
-    FST<Long> instance;
-    synchronized(this) {
-      instance = fstInstances.get(field.number);
-      if (instance == null) {
-        data.seek(entry.offset);
-        instance = new FST<>(data, PositiveIntOutputs.getSingleton());
-        ramBytesUsed.addAndGet(instance.sizeInBytes());
-        fstInstances.put(field.number, instance);
-      }
-    }
-    final BinaryDocValues docToOrds = getBinary(field);
-    final FST<Long> fst = instance;
-    
-    // per-thread resources
-    final BytesReader in = fst.getBytesReader();
-    final Arc<Long> firstArc = new Arc<>();
-    final Arc<Long> scratchArc = new Arc<>();
-    final IntsRef scratchInts = new IntsRef();
-    final BytesRefFSTEnum<Long> fstEnum = new BytesRefFSTEnum<>(fst);
-    final BytesRef ref = new BytesRef();
-    final ByteArrayDataInput input = new ByteArrayDataInput();
-    return new SortedSetDocValues() {
-      long currentOrd;
-
-      @Override
-      public long nextOrd() {
-        if (input.eof()) {
-          return NO_MORE_ORDS;
-        } else {
-          currentOrd += input.readVLong();
-          return currentOrd;
-        }
-      }
-      
-      @Override
-      public void setDocument(int docID) {
-        docToOrds.get(docID, ref);
-        input.reset(ref.bytes, ref.offset, ref.length);
-        currentOrd = 0;
-      }
-
-      @Override
-      public void lookupOrd(long ord, BytesRef result) {
-        try {
-          in.setPosition(0);
-          fst.getFirstArc(firstArc);
-          IntsRef output = Util.getByOutput(fst, ord, in, firstArc, scratchArc, scratchInts);
-          result.bytes = new byte[output.length];
-          result.offset = 0;
-          result.length = 0;
-          Util.toBytesRef(output, result);
-        } catch (IOException bogus) {
-          throw new RuntimeException(bogus);
-        }
-      }
-
-      @Override
-      public long lookupTerm(BytesRef key) {
-        try {
-          InputOutput<Long> o = fstEnum.seekCeil(key);
-          if (o == null) {
-            return -getValueCount()-1;
-          } else if (o.input.equals(key)) {
-            return o.output.intValue();
-          } else {
-            return -o.output-1;
-          }
-        } catch (IOException bogus) {
-          throw new RuntimeException(bogus);
-        }
-      }
-
-      @Override
-      public long getValueCount() {
-        return entry.numOrds;
-      }
-
-      @Override
-      public TermsEnum termsEnum() {
-        return new FSTTermsEnum(fst);
-      }
-    };
-  }
-  
-  private Bits getMissingBits(int fieldNumber, final long offset, final long length)  {
-    if (offset == -1) {
-      return new Bits.MatchAllBits(maxDoc);
-    } else {
-      Bits instance;
-      synchronized(this) {
-        instance = docsWithFieldInstances.get(fieldNumber);
-        if (instance == null) {
-          IndexInput data = this.data.clone();
-          data.seek(offset);
-          Debug.Assert( length % 8 == 0;
-          long bits[] = new long[(int) length >> 3];
-          for (int i = 0; i < bits.length; i++) {
-            bits[i] = data.readLong();
-          }
-          instance = new FixedBitSet(bits, maxDoc);
-          docsWithFieldInstances.put(fieldNumber, instance);
-        }
-      }
-      return instance;
-    }
-  }
-  
-  @Override
-  public Bits getDocsWithField(FieldInfo field)  {
-    switch(field.getDocValuesType()) {
-      case SORTED_SET:
-        return DocValues.docsWithValue(getSortedSet(field), maxDoc);
-      case SORTED:
-        return DocValues.docsWithValue(getSorted(field), maxDoc);
-      case BINARY:
-        BinaryEntry be = binaries.get(field.number);
-        return getMissingBits(field.number, be.missingOffset, be.missingBytes);
-      case NUMERIC:
-        NumericEntry ne = numerics.get(field.number);
-        return getMissingBits(field.number, ne.missingOffset, ne.missingBytes);
-      default: 
-        throw new Debug.Assert(ionError();
-    }
-  }
-
-  @Override
-  public void close()  {
-    data.close();
-  }
-  
-  static class NumericEntry {
-    long offset;
-    long missingOffset;
-    long missingBytes;
-    byte format;
-    int packedIntsVersion;
-  }
-  
-  static class BinaryEntry {
-    long offset;
-    long missingOffset;
-    long missingBytes;
-    long numBytes;
-    int minLength;
-    int maxLength;
-    int packedIntsVersion;
-    int blockSize;
-  }
-  
-  static class FSTEntry {
-    long offset;
-    long numOrds;
-  }
-  
-  // exposes FSTEnum directly as a TermsEnum: avoids binary-search next()
-  static class FSTTermsEnum extends TermsEnum {
-    final BytesRefFSTEnum<Long> in;
-    
-    // this is all for the complicated seek(ord)...
-    // maybe we should add a FSTEnum that supports this operation?
-    final FST<Long> fst;
-    final FST.BytesReader bytesReader;
-    final Arc<Long> firstArc = new Arc<>();
-    final Arc<Long> scratchArc = new Arc<>();
-    final IntsRef scratchInts = new IntsRef();
-    final BytesRef scratchBytes = new BytesRef();
-    
-    FSTTermsEnum(FST<Long> fst) {
-      this.fst = fst;
-      in = new BytesRefFSTEnum<>(fst);
-      bytesReader = fst.getBytesReader();
-    }
-
-    @Override
-    public BytesRef next()  {
-      InputOutput<Long> io = in.next();
-      if (io == null) {
-        return null;
-      } else {
-        return io.input;
-      }
-    }
-
-    @Override
-    public Comparator<BytesRef> getComparator() {
-      return BytesRef.getUTF8SortedAsUnicodeComparator();
-    }
-
-    @Override
-    public SeekStatus seekCeil(BytesRef text)  {
-      if (in.seekCeil(text) == null) {
-        return SeekStatus.END;
-      } else if (term().equals(text)) {
-        // TODO: add SeekStatus to FSTEnum like in https://issues.apache.org/jira/browse/LUCENE-3729
-        // to remove this comparision?
-        return SeekStatus.FOUND;
-      } else {
-        return SeekStatus.NOT_FOUND;
-      }
-    }
-
-    @Override
-    public bool seekExact(BytesRef text)  {
-      if (in.seekExact(text) == null) {
-        return false;
-      } else {
-        return true;
-      }
-    }
-
-    @Override
-    public void seekExact(long ord)  {
-      // TODO: would be better to make this simpler and faster.
-      // but we dont want to introduce a bug that corrupts our enum state!
-      bytesReader.setPosition(0);
-      fst.getFirstArc(firstArc);
-      IntsRef output = Util.getByOutput(fst, ord, bytesReader, firstArc, scratchArc, scratchInts);
-      scratchBytes.bytes = new byte[output.length];
-      scratchBytes.offset = 0;
-      scratchBytes.length = 0;
-      Util.toBytesRef(output, scratchBytes);
-      // TODO: we could do this lazily, better to try to push into FSTEnum though?
-      in.seekExact(scratchBytes);
-    }
-
-    @Override
-    public BytesRef term()  {
-      return in.current().input;
-    }
-
-    @Override
-    public long ord()  {
-      return in.current().output;
-    }
-
-    @Override
-    public int docFreq()  {
-      throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public long totalTermFreq()  {
-      throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public DocsEnum docs(Bits liveDocs, DocsEnum reuse, int flags)  {
-      throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, int flags)  {
-      throw new UnsupportedOperationException();
-    }
-  }
-}
+/*
+	 * Licensed to the Apache Software Foundation (ASF) under one or more
+	 * contributor license agreements.  See the NOTICE file distributed with
+	 * this work for additional information regarding copyright ownership.
+	 * The ASF licenses this file to You under the Apache License, Version 2.0
+	 * (the "License"); you may not use this file except in compliance with
+	 * the License.  You may obtain a copy of the License at
+	 *
+	 *     http://www.apache.org/licenses/LICENSE-2.0
+	 *
+	 * Unless required by applicable law or agreed to in writing, software
+	 * distributed under the License is distributed on an "AS IS" BASIS,
+	 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+	 * See the License for the specific language governing permissions and
+	 * limitations under the License.
+	 */
+
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using Lucene.Net.Codecs.Sep;
+using org.apache.lucene.codecs.memory;
+
+namespace Lucene.Net.Codecs.Memory
+{
+    /// <summary>
+	/// Reader for <seealso cref="MemoryDocValuesFormat"/>
+	/// </summary>
+	internal class MemoryDocValuesProducer : DocValuesProducer
+	{
+	  // metadata maps (just file pointers and minimal stuff)
+	  private readonly IDictionary<int?, NumericEntry> numerics;
+	  private readonly IDictionary<int?, BinaryEntry> binaries;
+	  private readonly IDictionary<int?, FSTEntry> fsts;
+	  private readonly IndexInput data;
+
+	  // ram instances we have already loaded
+	  private readonly IDictionary<int?, NumericDocValues> numericInstances = new Dictionary<int?, NumericDocValues>();
+	  private readonly IDictionary<int?, BinaryDocValues> binaryInstances = new Dictionary<int?, BinaryDocValues>();
+	  private readonly IDictionary<int?, FST<long?>> fstInstances = new Dictionary<int?, FST<long?>>();
+	  private readonly IDictionary<int?, Bits> docsWithFieldInstances = new Dictionary<int?, Bits>();
+
+	  private readonly int maxDoc;
+	  private readonly AtomicLong ramBytesUsed_Renamed;
+	  private readonly int version;
+
+	  internal const sbyte NUMBER = 0;
+	  internal const sbyte BYTES = 1;
+	  internal const sbyte org;
+
+	  internal const int BLOCK_SIZE = 4096;
+
+	  internal const sbyte DELTA_COMPRESSED = 0;
+	  internal const sbyte TABLE_COMPRESSED = 1;
+	  internal const sbyte UNCOMPRESSED = 2;
+	  internal const sbyte GCD_COMPRESSED = 3;
+
+	  internal const int VERSION_START = 0;
+	  internal const int VERSION_GCD_COMPRESSION = 1;
+	  internal const int VERSION_CHECKSUM = 2;
+	  internal const int VERSION_CURRENT = VERSION_CHECKSUM;
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: MemoryDocValuesProducer(org.apache.lucene.index.SegmentReadState state, String dataCodec, String dataExtension, String metaCodec, String metaExtension) throws java.io.IOException
+	  internal MemoryDocValuesProducer(SegmentReadState state, string dataCodec, string dataExtension, string metaCodec, string metaExtension)
+	  {
+		maxDoc = state.segmentInfo.DocCount;
+		string metaName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, metaExtension);
+		// read in the entries from the metadata file.
+		ChecksumIndexInput @in = state.directory.openChecksumInput(metaName, state.context);
+		bool success = false;
+		try
+		{
+		  version = CodecUtil.checkHeader(@in, metaCodec, VERSION_START, VERSION_CURRENT);
+		  numerics = new Dictionary<>();
+		  binaries = new Dictionary<>();
+		  fsts = new Dictionary<>();
+		  readFields(@in, state.fieldInfos);
+		  if (version >= VERSION_CHECKSUM)
+		  {
+			CodecUtil.checkFooter(@in);
+		  }
+		  else
+		  {
+			CodecUtil.checkEOF(@in);
+		  }
+		  ramBytesUsed_Renamed = new AtomicLong(RamUsageEstimator.shallowSizeOfInstance(this.GetType()));
+		  success = true;
+		}
+		finally
+		{
+		  if (success)
+		  {
+			IOUtils.close(@in);
+		  }
+		  else
+		  {
+			IOUtils.closeWhileHandlingException(@in);
+		  }
+		}
+
+		success = false;
+		try
+		{
+		  string dataName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, dataExtension);
+		  data = state.directory.openInput(dataName, state.context);
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final int version2 = org.apache.lucene.codecs.CodecUtil.checkHeader(data, dataCodec, VERSION_START, VERSION_CURRENT);
+		  int version2 = CodecUtil.checkHeader(data, dataCodec, VERSION_START, VERSION_CURRENT);
+		  if (version != version2)
+		  {
+			throw new CorruptIndexException("Format versions mismatch");
+		  }
+
+		  success = true;
+		}
+		finally
+		{
+		  if (!success)
+		  {
+			IOUtils.closeWhileHandlingException(this.data);
+		  }
+		}
+	  }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: private void readFields(org.apache.lucene.store.IndexInput meta, org.apache.lucene.index.FieldInfos infos) throws java.io.IOException
+	  private void readFields(IndexInput meta, FieldInfos infos)
+	  {
+		int fieldNumber = meta.readVInt();
+		while (fieldNumber != -1)
+		{
+		  int fieldType = meta.readByte();
+		  if (fieldType == NUMBER)
+		  {
+			NumericEntry entry = new NumericEntry();
+			entry.offset = meta.readLong();
+			entry.missingOffset = meta.readLong();
+			if (entry.missingOffset != -1)
+			{
+			  entry.missingBytes = meta.readLong();
+			}
+			else
+			{
+			  entry.missingBytes = 0;
+			}
+			entry.format = meta.readByte();
+			switch (entry.format)
+			{
+			  case DELTA_COMPRESSED:
+			  case TABLE_COMPRESSED:
+			  case GCD_COMPRESSED:
+			  case UNCOMPRESSED:
+				   break;
+			  default:
+				   throw new CorruptIndexException("Unknown format: " + entry.format + ", input=" + meta);
+			}
+			if (entry.format != UNCOMPRESSED)
+			{
+			  entry.packedIntsVersion = meta.readVInt();
+			}
+			numerics[fieldNumber] = entry;
+		  }
+		  else if (fieldType == BYTES)
+		  {
+			BinaryEntry entry = new BinaryEntry();
+			entry.offset = meta.readLong();
+			entry.numBytes = meta.readLong();
+			entry.missingOffset = meta.readLong();
+			if (entry.missingOffset != -1)
+			{
+			  entry.missingBytes = meta.readLong();
+			}
+			else
+			{
+			  entry.missingBytes = 0;
+			}
+			entry.minLength = meta.readVInt();
+			entry.maxLength = meta.readVInt();
+			if (entry.minLength != entry.maxLength)
+			{
+			  entry.packedIntsVersion = meta.readVInt();
+			  entry.blockSize = meta.readVInt();
+			}
+			binaries[fieldNumber] = entry;
+		  }
+		  else if (fieldType == FST)
+		  {
+			FSTEntry entry = new FSTEntry();
+			entry.offset = meta.readLong();
+			entry.numOrds = meta.readVLong();
+			fsts[fieldNumber] = entry;
+		  }
+		  else
+		  {
+			throw new CorruptIndexException("invalid entry type: " + fieldType + ", input=" + meta);
+		  }
+		  fieldNumber = meta.readVInt();
+		}
+	  }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public synchronized org.apache.lucene.index.NumericDocValues getNumeric(org.apache.lucene.index.FieldInfo field) throws java.io.IOException
+	  public override NumericDocValues getNumeric(FieldInfo field)
+	  {
+		  lock (this)
+		  {
+			NumericDocValues instance = numericInstances[field.number];
+			if (instance == null)
+			{
+			  instance = loadNumeric(field);
+			  numericInstances[field.number] = instance;
+			}
+			return instance;
+		  }
+	  }
+
+	  public override long ramBytesUsed()
+	  {
+		return ramBytesUsed_Renamed.get();
+	  }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public void checkIntegrity() throws java.io.IOException
+	  public override void checkIntegrity()
+	  {
+		if (version >= VERSION_CHECKSUM)
+		{
+		  CodecUtil.checksumEntireFile(data);
+		}
+	  }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: private org.apache.lucene.index.NumericDocValues loadNumeric(org.apache.lucene.index.FieldInfo field) throws java.io.IOException
+	  private NumericDocValues loadNumeric(FieldInfo field)
+	  {
+		NumericEntry entry = numerics[field.number];
+		data.seek(entry.offset + entry.missingBytes);
+		switch (entry.format)
+		{
+		  case TABLE_COMPRESSED:
+			int size = data.readVInt();
+			if (size > 256)
+			{
+			  throw new CorruptIndexException("TABLE_COMPRESSED cannot have more than 256 distinct values, input=" + data);
+			}
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final long decode[] = new long[size];
+			long[] decode = new long[size];
+			for (int i = 0; i < decode.Length; i++)
+			{
+			  decode[i] = data.readLong();
+			}
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final int formatID = data.readVInt();
+			int formatID = data.readVInt();
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final int bitsPerValue = data.readVInt();
+			int bitsPerValue = data.readVInt();
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final org.apache.lucene.util.packed.PackedInts.Reader ordsReader = org.apache.lucene.util.packed.PackedInts.getReaderNoHeader(data, org.apache.lucene.util.packed.PackedInts.Format.byId(formatID), entry.packedIntsVersion, maxDoc, bitsPerValue);
+			PackedInts.Reader ordsReader = PackedInts.getReaderNoHeader(data, PackedInts.Format.byId(formatID), entry.packedIntsVersion, maxDoc, bitsPerValue);
+			ramBytesUsed_Renamed.addAndGet(RamUsageEstimator.sizeOf(decode) + ordsReader.ramBytesUsed());
+			return new NumericDocValuesAnonymousInnerClassHelper(this, decode, ordsReader);
+		  case DELTA_COMPRESSED:
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final int blockSize = data.readVInt();
+			int blockSize = data.readVInt();
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final org.apache.lucene.util.packed.BlockPackedReader reader = new org.apache.lucene.util.packed.BlockPackedReader(data, entry.packedIntsVersion, blockSize, maxDoc, false);
+			BlockPackedReader reader = new BlockPackedReader(data, entry.packedIntsVersion, blockSize, maxDoc, false);
+			ramBytesUsed_Renamed.addAndGet(reader.ramBytesUsed());
+			return reader;
+		  case UNCOMPRESSED:
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final byte bytes[] = new byte[maxDoc];
+			sbyte[] bytes = new sbyte[maxDoc];
+			data.readBytes(bytes, 0, bytes.Length);
+			ramBytesUsed_Renamed.addAndGet(RamUsageEstimator.sizeOf(bytes));
+			return new NumericDocValuesAnonymousInnerClassHelper2(this, bytes);
+		  case GCD_COMPRESSED:
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final long min = data.readLong();
+			long min = data.readLong();
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final long mult = data.readLong();
+			long mult = data.readLong();
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final int quotientBlockSize = data.readVInt();
+			int quotientBlockSize = data.readVInt();
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final org.apache.lucene.util.packed.BlockPackedReader quotientReader = new org.apache.lucene.util.packed.BlockPackedReader(data, entry.packedIntsVersion, quotientBlockSize, maxDoc, false);
+			BlockPackedReader quotientReader = new BlockPackedReader(data, entry.packedIntsVersion, quotientBlockSize, maxDoc, false);
+			ramBytesUsed_Renamed.addAndGet(quotientReader.ramBytesUsed());
+			return new NumericDocValuesAnonymousInnerClassHelper3(this, min, mult, quotientReader);
+		  default:
+			throw new AssertionError();
+		}
+	  }
+
+	  private class NumericDocValuesAnonymousInnerClassHelper : NumericDocValues
+	  {
+		  private readonly MemoryDocValuesProducer outerInstance;
+
+		  private long[] decode;
+		  private IntIndexInput.Reader ordsReader;
+
+		  public NumericDocValuesAnonymousInnerClassHelper(MemoryDocValuesProducer outerInstance, long[] decode, IntIndexInput.Reader ordsReader)
+		  {
+			  this.outerInstance = outerInstance;
+			  this.decode = decode;
+			  this.ordsReader = ordsReader;
+		  }
+
+		  public override long get(int docID)
+		  {
+			return decode[(int)ordsReader.get(docID)];
+		  }
+	  }
+
+	  private class NumericDocValuesAnonymousInnerClassHelper2 : NumericDocValues
+	  {
+		  private readonly MemoryDocValuesProducer outerInstance;
+
+		  private sbyte[] bytes;
+
+		  public NumericDocValuesAnonymousInnerClassHelper2(MemoryDocValuesProducer outerInstance, sbyte[] bytes)
+		  {
+			  this.outerInstance = outerInstance;
+			  this.bytes = bytes;
+		  }
+
+		  public override long get(int docID)
+		  {
+			return bytes[docID];
+		  }
+	  }
+
+	  private class NumericDocValuesAnonymousInnerClassHelper3 : NumericDocValues
+	  {
+		  private readonly MemoryDocValuesProducer outerInstance;
+
+		  private long min;
+		  private long mult;
+		  private BlockPackedReader quotientReader;
+
+		  public NumericDocValuesAnonymousInnerClassHelper3(MemoryDocValuesProducer outerInstance, long min, long mult, BlockPackedReader quotientReader)
+		  {
+			  this.outerInstance = outerInstance;
+			  this.min = min;
+			  this.mult = mult;
+			  this.quotientReader = quotientReader;
+		  }
+
+		  public override long get(int docID)
+		  {
+			return min + mult * quotientReader.get(docID);
+		  }
+	  }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public synchronized org.apache.lucene.index.BinaryDocValues getBinary(org.apache.lucene.index.FieldInfo field) throws java.io.IOException
+	  public override BinaryDocValues getBinary(FieldInfo field)
+	  {
+		  lock (this)
+		  {
+			BinaryDocValues instance = binaryInstances[field.number];
+			if (instance == null)
+			{
+			  instance = loadBinary(field);
+			  binaryInstances[field.number] = instance;
+			}
+			return instance;
+		  }
+	  }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: private org.apache.lucene.index.BinaryDocValues loadBinary(org.apache.lucene.index.FieldInfo field) throws java.io.IOException
+	  private BinaryDocValues loadBinary(FieldInfo field)
+	  {
+		BinaryEntry entry = binaries[field.number];
+		data.seek(entry.offset);
+		PagedBytes bytes = new PagedBytes(16);
+		bytes.copy(data, entry.numBytes);
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final org.apache.lucene.util.PagedBytes.Reader bytesReader = bytes.freeze(true);
+		PagedBytes.Reader bytesReader = bytes.freeze(true);
+		if (entry.minLength == entry.maxLength)
+		{
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final int fixedLength = entry.minLength;
+		  int fixedLength = entry.minLength;
+		  ramBytesUsed_Renamed.addAndGet(bytes.ramBytesUsed());
+		  return new BinaryDocValuesAnonymousInnerClassHelper(this, bytesReader, fixedLength);
+		}
+		else
+		{
+		  data.seek(data.FilePointer + entry.missingBytes);
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final org.apache.lucene.util.packed.MonotonicBlockPackedReader addresses = new org.apache.lucene.util.packed.MonotonicBlockPackedReader(data, entry.packedIntsVersion, entry.blockSize, maxDoc, false);
+		  MonotonicBlockPackedReader addresses = new MonotonicBlockPackedReader(data, entry.packedIntsVersion, entry.blockSize, maxDoc, false);
+		  ramBytesUsed_Renamed.addAndGet(bytes.ramBytesUsed() + addresses.ramBytesUsed());
+		  return new BinaryDocValuesAnonymousInnerClassHelper2(this, bytesReader, addresses);
+		}
+	  }
+
+	  private class BinaryDocValuesAnonymousInnerClassHelper : BinaryDocValues
+	  {
+		  private readonly MemoryDocValuesProducer outerInstance;
+
+		  private IntIndexInput.Reader bytesReader;
+		  private int fixedLength;
+
+		  public BinaryDocValuesAnonymousInnerClassHelper(MemoryDocValuesProducer outerInstance, IntIndexInput.Reader bytesReader, int fixedLength)
+		  {
+			  this.outerInstance = outerInstance;
+			  this.bytesReader = bytesReader;
+			  this.fixedLength = fixedLength;
+		  }
+
+		  public override void get(int docID, BytesRef result)
+		  {
+			bytesReader.fillSlice(result, fixedLength * (long)docID, fixedLength);
+		  }
+	  }
+
+	  private class BinaryDocValuesAnonymousInnerClassHelper2 : BinaryDocValues
+	  {
+		  private readonly MemoryDocValuesProducer outerInstance;
+
+		  private IntIndexInput.Reader bytesReader;
+		  private MonotonicBlockPackedReader addresses;
+
+		  public BinaryDocValuesAnonymousInnerClassHelper2(MemoryDocValuesProducer outerInstance, IntIndexInput.Reader bytesReader, MonotonicBlockPackedReader addresses)
+		  {
+			  this.outerInstance = outerInstance;
+			  this.bytesReader = bytesReader;
+			  this.addresses = addresses;
+		  }
+
+		  public override void get(int docID, BytesRef result)
+		  {
+			long startAddress = docID == 0 ? 0 : addresses.get(docID - 1);
+			long endAddress = addresses.get(docID);
+			bytesReader.fillSlice(result, startAddress, (int)(endAddress - startAddress));
+		  }
+	  }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public org.apache.lucene.index.SortedDocValues getSorted(org.apache.lucene.index.FieldInfo field) throws java.io.IOException
+	  public override SortedDocValues getSorted(FieldInfo field)
+	  {
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final FSTEntry entry = fsts.get(field.number);
+		FSTEntry entry = fsts[field.number];
+		if (entry.numOrds == 0)
+		{
+		  return DocValues.EMPTY_SORTED;
+		}
+		FST<long?> instance;
+		lock (this)
+		{
+		  instance = fstInstances[field.number];
+		  if (instance == null)
+		  {
+			data.seek(entry.offset);
+			instance = new FST<>(data, PositiveIntOutputs.Singleton);
+			ramBytesUsed_Renamed.addAndGet(instance.sizeInBytes());
+			fstInstances[field.number] = instance;
+		  }
+		}
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final org.apache.lucene.index.NumericDocValues docToOrd = getNumeric(field);
+		NumericDocValues docToOrd = getNumeric(field);
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final org.apache.lucene.util.fst.FST<Long> fst = instance;
+		FST<long?> fst = instance;
+
+		// per-thread resources
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final org.apache.lucene.util.fst.FST.BytesReader in = fst.getBytesReader();
+		FST.BytesReader @in = fst.BytesReader;
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final org.apache.lucene.util.fst.FST.Arc<Long> firstArc = new org.apache.lucene.util.fst.FST.Arc<>();
+		FST.Arc<long?> firstArc = new FST.Arc<long?>();
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final org.apache.lucene.util.fst.FST.Arc<Long> scratchArc = new org.apache.lucene.util.fst.FST.Arc<>();
+		FST.Arc<long?> scratchArc = new FST.Arc<long?>();
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final org.apache.lucene.util.IntsRef scratchInts = new org.apache.lucene.util.IntsRef();
+		IntsRef scratchInts = new IntsRef();
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final org.apache.lucene.util.fst.BytesRefFSTEnum<Long> fstEnum = new org.apache.lucene.util.fst.BytesRefFSTEnum<>(fst);
+		BytesRefFSTEnum<long?> fstEnum = new BytesRefFSTEnum<long?>(fst);
+
+		return new SortedDocValuesAnonymousInnerClassHelper(this, entry, docToOrd, fst, @in, firstArc, scratchArc, scratchInts, fstEnum);
+	  }
+
+	  private class SortedDocValuesAnonymousInnerClassHelper : SortedDocValues
+	  {
+		  private readonly MemoryDocValuesProducer outerInstance;
+
+		  private MemoryDocValuesProducer.FSTEntry entry;
+		  private NumericDocValues docToOrd;
+//JAVA TO C# CONVERTER TODO TASK: Java wildcard generics are not converted to .NET:
+//ORIGINAL LINE: private org.apache.lucene.util.fst.FST<long?> fst;
+		  private FST<long?> fst;
+		  private FST.BytesReader @in;
+//JAVA TO C# CONVERTER TODO TASK: Java wildcard generics are not converted to .NET:
+//ORIGINAL LINE: private org.apache.lucene.util.fst.FST.Arc<long?> firstArc;
+		  private FST.Arc<long?> firstArc;
+//JAVA TO C# CONVERTER TODO TASK: Java wildcard generics are not converted to .NET:
+//ORIGINAL LINE: private org.apache.lucene.util.fst.FST.Arc<long?> scratchArc;
+		  private FST.Arc<long?> scratchArc;
+		  private IntsRef scratchInts;
+//JAVA TO C# CONVERTER TODO TASK: Java wildcard generics are not converted to .NET:
+//ORIGINAL LINE: private org.apache.lucene.util.fst.BytesRefFSTEnum<long?> fstEnum;
+		  private BytesRefFSTEnum<long?> fstEnum;
+
+		  public SortedDocValuesAnonymousInnerClassHelper<T1, T2, T3, T4>(MemoryDocValuesProducer outerInstance, org.MemoryDocValuesProducer.FSTEntry entry, NumericDocValues docToOrd, FST<T1> fst, FST.BytesReader @in, FST.Arc<T2> firstArc, FST.Arc<T3> scratchArc, IntsRef scratchInts, BytesRefFSTEnum<T4> fstEnum)
+		  {
+			  this.outerInstance = outerInstance;
+			  this.entry = entry;
+			  this.docToOrd = docToOrd;
+			  this.fst = fst;
+			  this.@in = @in;
+			  this.firstArc = firstArc;
+			  this.scratchArc = scratchArc;
+			  this.scratchInts = scratchInts;
+			  this.fstEnum = fstEnum;
+		  }
+
+		  public override int getOrd(int docID)
+		  {
+			return (int) docToOrd.get(docID);
+		  }
+
+		  public override void lookupOrd(int ord, BytesRef result)
+		  {
+			try
+			{
+			  @in.Position = 0;
+			  fst.getFirstArc(firstArc);
+			  IntsRef output = Util.getByOutput(fst, ord, @in, firstArc, scratchArc, scratchInts);
+			  result.bytes = new sbyte[output.length];
+			  result.offset = 0;
+			  result.length = 0;
+			  Util.toBytesRef(output, result);
+			}
+			catch (IOException bogus)
+			{
+			  throw new Exception(bogus);
+			}
+		  }
+
+		  public override int lookupTerm(BytesRef key)
+		  {
+			try
+			{
+			  BytesRefFSTEnum.InputOutput<long?> o = fstEnum.seekCeil(key);
+			  if (o == null)
+			  {
+				return -ValueCount - 1;
+			  }
+			  else if (o.input.Equals(key))
+			  {
+				return (int)o.output;
+			  }
+			  else
+			  {
+				return (int) -o.output - 1;
+			  }
+			}
+			catch (IOException bogus)
+			{
+			  throw new Exception(bogus);
+			}
+		  }
+
+		  public override int ValueCount
+		  {
+			  get
+			  {
+				return (int)entry.numOrds;
+			  }
+		  }
+
+		  public override TermsEnum termsEnum()
+		  {
+			return new FSTTermsEnum(fst);
+		  }
+	  }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public org.apache.lucene.index.SortedSetDocValues getSortedSet(org.apache.lucene.index.FieldInfo field) throws java.io.IOException
+	  public override SortedSetDocValues getSortedSet(FieldInfo field)
+	  {
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final FSTEntry entry = fsts.get(field.number);
+		FSTEntry entry = fsts[field.number];
+		if (entry.numOrds == 0)
+		{
+		  return DocValues.EMPTY_SORTED_SET; // empty FST!
+		}
+		FST<long?> instance;
+		lock (this)
+		{
+		  instance = fstInstances[field.number];
+		  if (instance == null)
+		  {
+			data.seek(entry.offset);
+			instance = new FST<>(data, PositiveIntOutputs.Singleton);
+			ramBytesUsed_Renamed.addAndGet(instance.sizeInBytes());
+			fstInstances[field.number] = instance;
+		  }
+		}
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final org.apache.lucene.index.BinaryDocValues docToOrds = getBinary(field);
+		BinaryDocValues docToOrds = getBinary(field);
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final org.apache.lucene.util.fst.FST<Long> fst = instance;
+		FST<long?> fst = instance;
+
+		// per-thread resources
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final org.apache.lucene.util.fst.FST.BytesReader in = fst.getBytesReader();
+		FST.BytesReader @in = fst.BytesReader;
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final org.apache.lucene.util.fst.FST.Arc<Long> firstArc = new org.apache.lucene.util.fst.FST.Arc<>();
+		FST.Arc<long?> firstArc = new FST.Arc<long?>();
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final org.apache.lucene.util.fst.FST.Arc<Long> scratchArc = new org.apache.lucene.util.fst.FST.Arc<>();
+		FST.Arc<long?> scratchArc = new FST.Arc<long?>();
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final org.apache.lucene.util.IntsRef scratchInts = new org.apache.lucene.util.IntsRef();
+		IntsRef scratchInts = new IntsRef();
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final org.apache.lucene.util.fst.BytesRefFSTEnum<Long> fstEnum = new org.apache.lucene.util.fst.BytesRefFSTEnum<>(fst);
+		BytesRefFSTEnum<long?> fstEnum = new BytesRefFSTEnum<long?>(fst);
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final org.apache.lucene.util.BytesRef ref = new org.apache.lucene.util.BytesRef();
+		BytesRef @ref = new BytesRef();
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final org.apache.lucene.store.ByteArrayDataInput input = new org.apache.lucene.store.ByteArrayDataInput();
+		ByteArrayDataInput input = new ByteArrayDataInput();
+		return new SortedSetDocValuesAnonymousInnerClassHelper(this, entry, docToOrds, fst, @in, firstArc, scratchArc, scratchInts, fstEnum, @ref, input);
+	  }
+
+	  private class SortedSetDocValuesAnonymousInnerClassHelper : SortedSetDocValues
+	  {
+		  private readonly MemoryDocValuesProducer outerInstance;
+
+		  private MemoryDocValuesProducer.FSTEntry entry;
+		  private BinaryDocValues docToOrds;
+//JAVA TO C# CONVERTER TODO TASK: Java wildcard generics are not converted to .NET:
+//ORIGINAL LINE: private org.apache.lucene.util.fst.FST<long?> fst;
+		  private FST<long?> fst;
+		  private FST.BytesReader @in;
+//JAVA TO C# CONVERTER TODO TASK: Java wildcard generics are not converted to .NET:
+//ORIGINAL LINE: private org.apache.lucene.util.fst.FST.Arc<long?> firstArc;
+		  private FST.Arc<long?> firstArc;
+//JAVA TO C# CONVERTER TODO TASK: Java wildcard generics are not converted to .NET:
+//ORIGINAL LINE: private org.apache.lucene.util.fst.FST.Arc<long?> scratchArc;
+		  private FST.Arc<long?> scratchArc;
+		  private IntsRef scratchInts;
+//JAVA TO C# CONVERTER TODO TASK: Java wildcard generics are not converted to .NET:
+//ORIGINAL LINE: private org.apache.lucene.util.fst.BytesRefFSTEnum<long?> fstEnum;
+		  private BytesRefFSTEnum<long?> fstEnum;
+		  private BytesRef @ref;
+		  private ByteArrayDataInput input;
+
+		  public SortedSetDocValuesAnonymousInnerClassHelper<T1, T2, T3, T4>(MemoryDocValuesProducer outerInstance, org.MemoryDocValuesProducer.FSTEntry entry, BinaryDocValues docToOrds, FST<T1> fst, FST.BytesReader @in, FST.Arc<T2> firstArc, FST.Arc<T3> scratchArc, IntsRef scratchInts, BytesRefFSTEnum<T4> fstEnum, BytesRef @ref, ByteArrayDataInput input)
+		  {
+			  this.outerInstance = outerInstance;
+			  this.entry = entry;
+			  this.docToOrds = docToOrds;
+			  this.fst = fst;
+			  this.@in = @in;
+			  this.firstArc = firstArc;
+			  this.scratchArc = scratchArc;
+			  this.scratchInts = scratchInts;
+			  this.fstEnum = fstEnum;
+			  this.@ref = @ref;
+			  this.input = input;
+		  }
+
+		  internal long currentOrd;
+
+		  public override long nextOrd()
+		  {
+			if (input.eof())
+			{
+			  return NO_MORE_ORDS;
+			}
+			else
+			{
+			  currentOrd += input.readVLong();
+			  return currentOrd;
+			}
+		  }
+
+		  public override int Document
+		  {
+			  set
+			  {
+				docToOrds.get(value, @ref);
+				input.reset(@ref.bytes, @ref.offset, @ref.length);
+				currentOrd = 0;
+			  }
+		  }
+
+		  public override void lookupOrd(long ord, BytesRef result)
+		  {
+			try
+			{
+			  @in.Position = 0;
+			  fst.getFirstArc(firstArc);
+			  IntsRef output = Util.getByOutput(fst, ord, @in, firstArc, scratchArc, scratchInts);
+			  result.bytes = new sbyte[output.length];
+			  result.offset = 0;
+			  result.length = 0;
+			  Util.toBytesRef(output, result);
+			}
+			catch (IOException bogus)
+			{
+			  throw new Exception(bogus);
+			}
+		  }
+
+		  public override long lookupTerm(BytesRef key)
+		  {
+			try
+			{
+			  BytesRefFSTEnum.InputOutput<long?> o = fstEnum.seekCeil(key);
+			  if (o == null)
+			  {
+				return -ValueCount - 1;
+			  }
+			  else if (o.input.Equals(key))
+			  {
+				return (int)o.output;
+			  }
+			  else
+			  {
+				return -o.output - 1;
+			  }
+			}
+			catch (IOException bogus)
+			{
+			  throw new Exception(bogus);
+			}
+		  }
+
+		  public override long ValueCount
+		  {
+			  get
+			  {
+				return entry.numOrds;
+			  }
+		  }
+
+		  public override TermsEnum termsEnum()
+		  {
+			return new FSTTermsEnum(fst);
+		  }
+	  }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: private org.apache.lucene.util.Bits getMissingBits(int fieldNumber, final long offset, final long length) throws java.io.IOException
+//JAVA TO C# CONVERTER WARNING: 'final' parameters are not available in .NET:
+	  private Bits getMissingBits(int fieldNumber, long offset, long length)
+	  {
+		if (offset == -1)
+		{
+		  return new Bits.MatchAllBits(maxDoc);
+		}
+		else
+		{
+		  Bits instance;
+		  lock (this)
+		  {
+			instance = docsWithFieldInstances[fieldNumber];
+			if (instance == null)
+			{
+			  IndexInput data = this.data.clone();
+			  data.seek(offset);
+			  Debug.Assert(length % 8 == 0);
+			  long[] bits = new long[(int) length >> 3];
+			  for (int i = 0; i < bits.Length; i++)
+			  {
+				bits[i] = data.readLong();
+			  }
+			  instance = new FixedBitSet(bits, maxDoc);
+			  docsWithFieldInstances[fieldNumber] = instance;
+			}
+		  }
+		  return instance;
+		}
+	  }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public org.apache.lucene.util.Bits getDocsWithField(org.apache.lucene.index.FieldInfo field) throws java.io.IOException
+	  public override Bits getDocsWithField(FieldInfo field)
+	  {
+		switch (field.DocValuesType)
+		{
+		  case SORTED_SET:
+			return DocValues.docsWithValue(getSortedSet(field), maxDoc);
+		  case SORTED:
+			return DocValues.docsWithValue(getSorted(field), maxDoc);
+		  case BINARY:
+			BinaryEntry be = binaries[field.number];
+			return getMissingBits(field.number, be.missingOffset, be.missingBytes);
+		  case NUMERIC:
+			NumericEntry ne = numerics[field.number];
+			return getMissingBits(field.number, ne.missingOffset, ne.missingBytes);
+		  default:
+			throw new AssertionError();
+		}
+	  }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public void close() throws java.io.IOException
+	  public override void close()
+	  {
+		data.close();
+	  }
+
+	  internal class NumericEntry
+	  {
+		internal long offset;
+		internal long missingOffset;
+		internal long missingBytes;
+		internal sbyte format;
+		internal int packedIntsVersion;
+	  }
+
+	  internal class BinaryEntry
+	  {
+		internal long offset;
+		internal long missingOffset;
+		internal long missingBytes;
+		internal long numBytes;
+		internal int minLength;
+		internal int maxLength;
+		internal int packedIntsVersion;
+		internal int blockSize;
+	  }
+
+	  internal class FSTEntry
+	  {
+		internal long offset;
+		internal long numOrds;
+	  }
+
+	  // exposes FSTEnum directly as a TermsEnum: avoids binary-search next()
+	  internal class FSTTermsEnum : TermsEnum
+	  {
+		internal readonly BytesRefFSTEnum<long?> @in;
+
+		// this is all for the complicated seek(ord)...
+		// maybe we should add a FSTEnum that supports this operation?
+		internal readonly FST<long?> fst;
+		internal readonly FST.BytesReader bytesReader;
+		internal readonly FST.Arc<long?> firstArc = new FST.Arc<long?>();
+		internal readonly FST.Arc<long?> scratchArc = new FST.Arc<long?>();
+		internal readonly IntsRef scratchInts = new IntsRef();
+		internal readonly BytesRef scratchBytes = new BytesRef();
+
+		internal FSTTermsEnum(FST<long?> fst)
+		{
+		  this.fst = fst;
+		  @in = new BytesRefFSTEnum<>(fst);
+		  bytesReader = fst.BytesReader;
+		}
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public org.apache.lucene.util.BytesRef next() throws java.io.IOException
+		public override BytesRef next()
+		{
+		  BytesRefFSTEnum.InputOutput<long?> io = @in.next();
+		  if (io == null)
+		  {
+			return null;
+		  }
+		  else
+		  {
+			return io.input;
+		  }
+		}
+
+		public override IComparer<BytesRef> Comparator
+		{
+			get
+			{
+			  return BytesRef.UTF8SortedAsUnicodeComparator;
+			}
+		}
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public SeekStatus seekCeil(org.apache.lucene.util.BytesRef text) throws java.io.IOException
+		public override SeekStatus seekCeil(BytesRef text)
+		{
+		  if (@in.seekCeil(text) == null)
+		  {
+			return SeekStatus.END;
+		  }
+		  else if (term().Equals(text))
+		  {
+			// TODO: add SeekStatus to FSTEnum like in https://issues.apache.org/jira/browse/LUCENE-3729
+			// to remove this comparision?
+			return SeekStatus.FOUND;
+		  }
+		  else
+		  {
+			return SeekStatus.NOT_FOUND;
+		  }
+		}
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public boolean seekExact(org.apache.lucene.util.BytesRef text) throws java.io.IOException
+		public override bool seekExact(BytesRef text)
+		{
+		  if (@in.seekExact(text) == null)
+		  {
+			return false;
+		  }
+		  else
+		  {
+			return true;
+		  }
+		}
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public void seekExact(long ord) throws java.io.IOException
+		public override void seekExact(long ord)
+		{
+		  // TODO: would be better to make this simpler and faster.
+		  // but we dont want to introduce a bug that corrupts our enum state!
+		  bytesReader.Position = 0;
+		  fst.getFirstArc(firstArc);
+		  IntsRef output = Util.getByOutput(fst, ord, bytesReader, firstArc, scratchArc, scratchInts);
+		  scratchBytes.bytes = new sbyte[output.length];
+		  scratchBytes.offset = 0;
+		  scratchBytes.length = 0;
+		  Util.toBytesRef(output, scratchBytes);
+		  // TODO: we could do this lazily, better to try to push into FSTEnum though?
+		  @in.seekExact(scratchBytes);
+		}
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public org.apache.lucene.util.BytesRef term() throws java.io.IOException
+		public override BytesRef term()
+		{
+		  return @in.current().input;
+		}
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public long ord() throws java.io.IOException
+		public override long ord()
+		{
+		  return @in.current().output;
+		}
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public int docFreq() throws java.io.IOException
+		public override int docFreq()
+		{
+		  throw new System.NotSupportedException();
+		}
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public long totalTermFreq() throws java.io.IOException
+		public override long totalTermFreq()
+		{
+		  throw new System.NotSupportedException();
+		}
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public org.apache.lucene.index.DocsEnum docs(org.apache.lucene.util.Bits liveDocs, org.apache.lucene.index.DocsEnum reuse, int flags) throws java.io.IOException
+		public override DocsEnum docs(Bits liveDocs, DocsEnum reuse, int flags)
+		{
+		  throw new System.NotSupportedException();
+		}
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public org.apache.lucene.index.DocsAndPositionsEnum docsAndPositions(org.apache.lucene.util.Bits liveDocs, org.apache.lucene.index.DocsAndPositionsEnum reuse, int flags) throws java.io.IOException
+		public override DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, int flags)
+		{
+		  throw new System.NotSupportedException();
+		}
+	  }
+	}
+
+}
\ No newline at end of file


Mime
View raw message