lucene-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From Scott Ganyo <scott.ga...@eTapestry.com>
Subject Searcher/Reader/Writer Management
Date Tue, 02 Apr 2002 20:28:53 GMT
It seems that a lot of people run into the same set of problems with
maintaining readers, writers, and searchers in Lucene.  Part of the problem
comes from the fact that delete() is on a reader and part of it comes from
the need to keep a reader or search open as long as possible to avoid
open/close overhead.  Add to that the need for some applications (like mine)
to maintain multiple indexes and you have a pretty complex problem space.
Bottom line: Use of Lucene (especially in an online, interactive
application) doesn't seem as easy as it could be.

Anyway... I've attempted to address these issues in my application with a
control class.  And, I'm submitting what I have to lucene dev for review...
and consideration for possible inclusion in the core lucene framework.  I
tried to be fairly efficient and to handle all cases that I know about so
let me know if it fails in either case and what you think overall.

Thanks,
Scott

-------

import java.io.*;
import java.util.*;
import org.apache.lucene.index.*;
import org.apache.lucene.document.*;
import org.apache.lucene.search.*;
import org.apache.lucene.analysis.*;

/** Rules:
 *  Once created, searchers are valid until closed.
 *  If doing an update, searchers must not be created between close of the
reader
 *      that deletes and close of the writer that adds.
 *  Writers may be reused until Reader or Searcher is needed.
 *  Readers may be reused until Writer or Searcher is needed.
 *  Searchers may be reused until the index is changed.
 *  There may be only one of a Reader and/or Writer at a time.
 *  If you get it, release it.
 */
public class IndexAccessControl
{
    public static final Analyzer LUCENE_ANALYZER = new LuceneAnalyzer();
    private static final Map WRITER_PATHS = new HashMap(); // path ->
CheckoutInfo
    private static final Map SEARCHER_PATHS = new HashMap(); // path ->
Searcher
    private static final Map OLD_SEARCHERS = new HashMap(); // Searcher ->
CheckoutInfo
    
    /** get for adding documents. 
     * blocks: readers until released
     */
    public static IndexWriter getWriter(File path) throws IOException
    {
        IndexWriter writer = null;
        String sync = path.getAbsolutePath().intern();
        synchronized (sync) // sync on specific index
        {
            do
            {
                CheckoutInfo info = (CheckoutInfo)WRITER_PATHS.get(path);
                if (info != null) // may already have a writer, use it
                {
                    if (info.writer != null) // yup, have a writer
                    {
                        info.checkoutCount++;
                        writer = info.writer;
                    }
                    else // not a writer, it must be a reader, wait for it
to finish to try again
                    {
                        try
                        {
                            info.wait(); // wait for info to be released
                        }
                        catch (InterruptedException e)
                        {
                            // TODO: Will this ever happen?
                            e.printStackTrace();
                            return null;
                        }
                    }
                }
                else // no writer, create one
                {
                    boolean missing = !path.exists();
                    if (missing) path.mkdir();
                    writer = new IndexWriter(path, LUCENE_ANALYZER,
/*create*/missing);
                    writer.mergeFactor = 2;
                    info = new CheckoutInfo(writer);
                }
            }
            while (writer == null);
        }
        return writer;
    }
    
    public static void releaseWriter(File path, IndexWriter writer) throws
IOException
    {
        String sync = path.getAbsolutePath().intern();
        synchronized (sync) // sync on specific index
        {
            CheckoutInfo info = (CheckoutInfo)WRITER_PATHS.get(path);
            if (info != null && writer == info.writer) // writer was checked
out
            {
                if (info.checkoutCount > 1) // writer has other references
                {
                    info.checkoutCount--;
                    writer = null; // avoid close()
                }
                else // last reference to writer
                {
                    WRITER_PATHS.remove(path);
                    writer = info.writer;
                    info.notify(); // notify waiters to try again
                }
            }
        }
        // close the writer (unless it still has checkouts)
        if (writer != null) writer.close();
    }
    
    /** get for search. */
    public static Searcher getSearcher(File path) throws IOException
    {
        IndexSearcher is;
        String sync = path.getAbsolutePath().intern();
        synchronized (sync) // sync on specific index
        {
            CheckoutInfo info = (CheckoutInfo)SEARCHER_PATHS.get(path);
            if (info == null || IndexReader.lastModified(path) >
info.creationTime)
            {
                // need new searcher
                is = new IndexSearcher(IndexReader.open(path));
                info = (CheckoutInfo)SEARCHER_PATHS.put(path, new
CheckoutInfo(is));
                if (info != null)
                {
                    if (info.checkoutCount > 1) // searcher has other
references
                    {
                        info.checkoutCount--;
                        OLD_SEARCHERS.put(info.searcher, info);
                    }
                    else // last reference to searcher
                    {
                        info.searcher.close();
                    }
                }
            }
            else
            {
                // use existing searcher
                is = info.searcher;
                info.checkoutCount++;
            }
        }
        return is;
    }

    public static void releaseSearcher(File path, Searcher searcher) throws
IOException
    {
        String sync = path.getAbsolutePath().intern();
        synchronized (sync) // sync on specific index
        {
            CheckoutInfo info = (CheckoutInfo)SEARCHER_PATHS.get(path);
            if (info == null || searcher != info.searcher) // this isn't the
info we're looking for
            {
                info = (CheckoutInfo)OLD_SEARCHERS.get(searcher);
            }
            if (info != null) // found a searcher
            {
                if (info.checkoutCount > 1) // searcher has other references
                {
                    info.checkoutCount--;
                }
                else // last reference to searcher
                {
                    info.searcher.close();
                }
            }
            else // can't find searcher, just close it
            {
                searcher.close();
            }
        }
    }
    
    /** get for deleting documents. 
     * blocks: writers until released
     */
    public static IndexReader getReader(File path) throws IOException
    {
        IndexReader reader = null;
        String sync = path.getAbsolutePath().intern();
        synchronized (sync) // sync on specific index
        {
            do
            {
                CheckoutInfo info = (CheckoutInfo)WRITER_PATHS.get(path);
                if (info != null) // may already have a reader, use it
                {
                    if (info.reader != null) // yup, have a reader
                    {
                        info.checkoutCount++;
                        reader = info.reader;
                    }
                    else // not a reader, it must be a writer, wait for it
to finish to try again
                    {
                        try
                        {
                            info.wait(); // wait for info to be released
                        }
                        catch (InterruptedException e)
                        {
                            // TODO: Will this ever happen?
                            e.printStackTrace();
                            return null;
                        }
                    }
                }
                else // no reader, create one
                {
                    reader = IndexReader.open(path);
                    info = new CheckoutInfo(reader);
                }
            }
            while (reader == null);
        }
        return reader;
    }

    public static void releaseReader(File path, IndexReader reader) throws
IOException
    {
        String sync = path.getAbsolutePath().intern();
        synchronized (sync) // sync on specific index
        {
            CheckoutInfo info = (CheckoutInfo)WRITER_PATHS.get(path);
            if (info != null && reader == info.reader) // reader was checked
out
            {
                if (info.checkoutCount > 1) // reader has other references
                {
                    info.checkoutCount--;
                    reader = null; // avoid close()
                }
                else // last reference to reader
                {
                    WRITER_PATHS.remove(path);
                    reader = info.reader;
                    info.notify(); // notify waiters to try again
                }
            }
        }
        // close the reader (unless it still has checkouts)
        if (reader != null) reader.close();
    }
    
    /** used for updates to make sure nobody else grabs a writer or reader
between 
     *  release and get operations.
     */
    public static IndexWriter releaseReaderAndGetWriter(File path,
IndexReader reader) throws IOException
    {
        String sync = path.getAbsolutePath().intern();
        synchronized (sync) // sync on specific index
        {
            releaseReader(path, reader);
            return getWriter(path);
        }
    }
    
    private static class CheckoutInfo
    {
        CheckoutInfo(IndexWriter writer)
        {
            this.writer = writer;
        }
        CheckoutInfo(IndexReader reader)
        {
            this.reader = reader;
        }
        CheckoutInfo(IndexSearcher searcher)
        {
            this.searcher = searcher;
            this.creationTime = System.currentTimeMillis();
        }
        
        public IndexReader reader;
        public IndexWriter writer;
        public IndexSearcher searcher;
        public int checkoutCount = 1;
        public long creationTime;
    }
    
    /** no instances */
    private IndexAccessControl() { }
}

Mime
  • Unnamed multipart/alternative (inline, None, 0 bytes)
View raw message