lucene-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cutt...@apache.org
Subject cvs commit: jakarta-lucene/src/java/org/apache/lucene/store MMapDirectory.java
Date Mon, 04 Oct 2004 19:44:11 GMT
cutting     2004/10/04 12:44:11

  Modified:    src/java/org/apache/lucene/store MMapDirectory.java
  Log:
  Improved version by Paul Elschot that can handle files longer than 2^31.
  
  Revision  Changes    Path
  1.3       +128 -12   jakarta-lucene/src/java/org/apache/lucene/store/MMapDirectory.java
  
  Index: MMapDirectory.java
  ===================================================================
  RCS file: /home/cvs/jakarta-lucene/src/java/org/apache/lucene/store/MMapDirectory.java,v
  retrieving revision 1.2
  retrieving revision 1.3
  diff -u -r1.2 -r1.3
  --- MMapDirectory.java	28 Sep 2004 21:55:59 -0000	1.2
  +++ MMapDirectory.java	4 Oct 2004 19:44:11 -0000	1.3
  @@ -15,7 +15,7 @@
    * See the License for the specific language governing permissions and
    * limitations under the License.
    */
  -
  + 
   import java.io.IOException;
   import java.io.File;
   import java.io.RandomAccessFile;
  @@ -29,24 +29,17 @@
    * org.apache.lucene.FSDirectory.class set to
    * org.apache.lucene.store.MMapDirectory.  This will cause {@link
    * FSDirectory#getDirectory(File,boolean)} to return instances of this class.
  - *
  - * @author Doug Cutting
    */
   public class MMapDirectory extends FSDirectory {
   
  -  private class MMapIndexInput extends IndexInput {
  +  private static class MMapIndexInput extends IndexInput {
   
       private ByteBuffer buffer;
       private final long length;
   
  -    public MMapIndexInput(File file) throws IOException {
  -      RandomAccessFile raf = new RandomAccessFile(file, "r");
  -      try {
  +    private MMapIndexInput(RandomAccessFile raf) throws IOException {
           this.length = raf.length();
           this.buffer = raf.getChannel().map(MapMode.READ_ONLY, 0, length);
  -      } finally {
  -        raf.close();
  -      }
       }
   
       public byte readByte() throws IOException {
  @@ -77,11 +70,134 @@
       }
   
       public void close() throws IOException {}
  +  }
   
  +  /* Added class MultiMMapIndexInput, Paul Elschot.
  +   * Slightly adapted constructor of MMapIndexInput.
  +   * Licensed under the Apache License, Version 2.0.
  +   */
  +  private static class MultiMMapIndexInput extends IndexInput {
  +  
  +    private ByteBuffer[] buffers;
  +    private int[] bufSizes; // keep here, ByteBuffer.size() method is optional
  +  
  +    private final long length;
  +  
  +    private int curBufIndex;
  +    private final int maxBufSize;
  +  
  +    private ByteBuffer curBuf; // redundant for speed: buffers[curBufIndex]
  +    private int curAvail; // redundant for speed: (bufSizes[curBufIndex] - curBuf.position())
  +  
  +    
  +    public MultiMMapIndexInput(RandomAccessFile raf, int maxBufSize)
  +      throws IOException {
  +      this.length = raf.length();
  +      this.maxBufSize = maxBufSize;
  +      
  +      if (maxBufSize <= 0)
  +        throw new IllegalArgumentException("Non positive maxBufSize: "
  +                                           + maxBufSize);
  +      
  +      if ((length / maxBufSize) > Integer.MAX_VALUE)
  +        throw new IllegalArgumentException
  +          ("RandomAccessFile too big for maximum buffer size: "
  +           + raf.toString());
  +      
  +      int nrBuffers = (int) (length / maxBufSize);
  +      if ((nrBuffers * maxBufSize) < length) nrBuffers++;
  +      
  +      this.buffers = new ByteBuffer[nrBuffers];
  +      this.bufSizes = new int[nrBuffers];
  +      
  +      long bufferStart = 0;
  +      FileChannel rafc = raf.getChannel();
  +      for (int bufNr = 0; bufNr < nrBuffers; bufNr++) { 
  +        int bufSize = (length > (bufferStart + maxBufSize))
  +          ? maxBufSize
  +          : (int) (length - bufferStart);
  +        //System.out.println("mapping from: "+bufferStart+", size: "+bufSize);
  +        this.buffers[bufNr] = rafc.map(MapMode.READ_ONLY,bufferStart,bufSize);
  +        this.bufSizes[bufNr] = bufSize;
  +        bufferStart += bufSize;
  +      }
  +      seek(0L);
  +    }
  +  
  +    public byte readByte() throws IOException {
  +      // Performance might be improved by reading ahead into an array of
  +      // eg. 128 bytes and readByte() from there.
  +      if (curAvail == 0) {
  +        curBufIndex++;
  +        curBuf = buffers[curBufIndex];
  +        curBuf.position(0);      // index out of bounds when too many requested
  +        curAvail = bufSizes[curBufIndex];
  +      }
  +      curAvail--;
  +      return curBuf.get();
  +    }
  +  
  +    public void readBytes(byte[] b, int offset, int len) throws IOException {
  +      while (len > curAvail) {
  +        curBuf.get(b, offset, curAvail);
  +        len -= curAvail;
  +        offset += curAvail;
  +        curBufIndex++;
  +        curBuf = buffers[curBufIndex];
  +        curBuf.position(0);      // index out of bounds when too many requested
  +        curAvail = bufSizes[curBufIndex];
  +      }
  +      curBuf.get(b, offset, len);
  +      curAvail -= len;
  +    }
  +  
  +    public long getFilePointer() {
  +      return (curBufIndex * (long) maxBufSize) + curBuf.position();
  +    }
  +  
  +    public void seek(long pos) throws IOException {
  +      curBufIndex = (int) (pos / maxBufSize);
  +      curBuf = buffers[curBufIndex];
  +      int bufOffset = (int) (pos - (curBufIndex * maxBufSize));
  +      curBuf.position(bufOffset);
  +      curAvail = bufSizes[curBufIndex] - bufOffset;
  +    }
  +  
  +    public long length() {
  +      return length;
  +    }
  +  
  +    public Object clone() {
  +      MultiMMapIndexInput clone = (MultiMMapIndexInput)super.clone();
  +      clone.buffers = new ByteBuffer[buffers.length];
  +      // No need to clone bufSizes.
  +      // Since most clones will use only one buffer, duplicate() could also be
  +      // done lazy in clones, eg. when adapting curBuf.
  +      for (int bufNr = 0; bufNr < buffers.length; bufNr++) {
  +        clone.buffers[bufNr] = buffers[bufNr].duplicate();
  +      }
  +      try {
  +        clone.seek(getFilePointer());
  +      } catch(IOException ioe) {
  +        throw new RuntimeException(ioe);
  +      };
  +      return clone;
  +    }
  +  
  +    public void close() throws IOException {}
     }
  +  
  +  private final int MAX_BBUF = Integer.MAX_VALUE;
   
     public IndexInput openInput(String name) throws IOException {
  -    return new MMapIndexInput(new File(getFile(), name));
  +    File f =  new File(getFile(), name);
  +    RandomAccessFile raf = new RandomAccessFile(f, "r");
  +    try {
  +      return (raf.length() <= MAX_BBUF)
  +             ? (IndexInput) new MMapIndexInput(raf)
  +             : (IndexInput) new MultiMMapIndexInput(raf, MAX_BBUF);
  +    } finally {
  +      raf.close();
  +    }
     }
   }
  -
  
  
  

---------------------------------------------------------------------
To unsubscribe, e-mail: lucene-dev-unsubscribe@jakarta.apache.org
For additional commands, e-mail: lucene-dev-help@jakarta.apache.org


Mime
View raw message