lucene-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From Grant Ingersoll <gsing...@apache.org>
Subject Re: svn commit: r659635 - in /lucene/java/trunk: ./ contrib/miscellaneous/src/java/org/apache/lucene/misc/ contrib/miscellaneous/src/test/org/apache/lucene/misc/ contrib/queries/src/java/org/apache/lucene/search/ contrib/queries/src/test/org/apache/lucene/...
Date Fri, 23 May 2008 23:04:44 GMT
I'm getting:

common.compile-test:
    [javac] Compiling 5 source files to /lucene-clean/build/contrib/ 
misc/classes/test
    [javac] .../lucene-clean/contrib/miscellaneous/src/test/org/apache/ 
lucene/misc/ChainedFilterTest.java:92: cannot find symbol
    [javac] symbol  : class OldBitSetFilterWrapper
    [javac] location: class org.apache.lucene.misc.ChainedFilterTest
    [javac]       oldFilters[i] = new OldBitSetFilterWrapper(chain[i]);
    [javac]                           ^
    [javac] Note: Some input files use or override a deprecated API.
    [javac] Note: Recompile with -Xlint:deprecation for details.
    [javac] 1 error



When trying to run ant dist from the top.

svn info
Path: .
URL: https://svn.apache.org/repos/asf/lucene/java/trunk/contrib/miscellaneous
Repository Root: https://svn.apache.org/repos/asf
Repository UUID: 13f79535-47bb-0310-9956-ffa450edef68
Revision: 659700
Node Kind: directory
Schedule: normal
Last Changed Author: buschmi
Last Changed Rev: 659635
Last Changed Date: 2008-05-23 15:25:05 -0400 (Fri, 23 May 2008)


I'm up to date and I did a clean first.


-Grant

On May 23, 2008, at 3:25 PM, buschmi@apache.org wrote:

> Author: buschmi
> Date: Fri May 23 12:25:05 2008
> New Revision: 659635
>
> URL: http://svn.apache.org/viewvc?rev=659635&view=rev
> Log:
> LUCENE-1187: ChainedFilter and BooleanFilter now work with new  
> Filter API and DocIdSetIterator-based filters.
>
> Added:
>   lucene/java/trunk/src/java/org/apache/lucene/util/ 
> OpenBitSetDISI.java
>   lucene/java/trunk/src/test/org/apache/lucene/search/ 
> OldBitSetFilterWrapper.java
> Modified:
>   lucene/java/trunk/CHANGES.txt
>   lucene/java/trunk/contrib/miscellaneous/src/java/org/apache/lucene/ 
> misc/ChainedFilter.java
>   lucene/java/trunk/contrib/miscellaneous/src/test/org/apache/lucene/ 
> misc/ChainedFilterTest.java
>   lucene/java/trunk/contrib/queries/src/java/org/apache/lucene/ 
> search/BooleanFilter.java
>   lucene/java/trunk/contrib/queries/src/java/org/apache/lucene/ 
> search/DuplicateFilter.java
>   lucene/java/trunk/contrib/queries/src/java/org/apache/lucene/ 
> search/TermsFilter.java
>   lucene/java/trunk/contrib/queries/src/test/org/apache/lucene/ 
> search/BooleanFilterTest.java
>   lucene/java/trunk/src/java/org/apache/lucene/search/ 
> ParallelMultiSearcher.java
>   lucene/java/trunk/src/java/org/apache/lucene/search/Searchable.java
>   lucene/java/trunk/src/java/org/apache/lucene/search/Searcher.java
>
> Modified: lucene/java/trunk/CHANGES.txt
> URL: http://svn.apache.org/viewvc/lucene/java/trunk/CHANGES.txt?rev=659635&r1=659634&r2=659635&view=diff
> = 
> = 
> = 
> = 
> = 
> = 
> = 
> = 
> ======================================================================
> --- lucene/java/trunk/CHANGES.txt (original)
> +++ lucene/java/trunk/CHANGES.txt Fri May 23 12:25:05 2008
> @@ -163,6 +163,10 @@
>
> 13. LUCENE-1166: Decomposition tokenfilter for languages like German  
> and Swedish (Thomas Peuss via Grant Ingersoll)
>
> +14. LUCENE-1187: ChainedFilter and BooleanFilter now work with new  
> Filter API
> +    and DocIdSetIterator-based filters. Backwards-compatibility  
> with old
> +    BitSet-based filters is ensured. (Paul Elschot via Michael Busch)
> +
> Optimizations
>
> 1. LUCENE-705: When building a compound file, use
>
> Modified: lucene/java/trunk/contrib/miscellaneous/src/java/org/ 
> apache/lucene/misc/ChainedFilter.java
> URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/miscellaneous/src/java/org/apache/lucene/misc/ChainedFilter.java?rev=659635&r1=659634&r2=659635&view=diff
> = 
> = 
> = 
> = 
> = 
> = 
> = 
> = 
> ======================================================================
> --- lucene/java/trunk/contrib/miscellaneous/src/java/org/apache/ 
> lucene/misc/ChainedFilter.java (original)
> +++ lucene/java/trunk/contrib/miscellaneous/src/java/org/apache/ 
> lucene/misc/ChainedFilter.java Fri May 23 12:25:05 2008
> @@ -58,7 +58,11 @@
> import org.apache.lucene.search.Filter;
>
> import java.io.IOException;
> -import java.util.BitSet;
> +import org.apache.lucene.search.DocIdSet;
> +import org.apache.lucene.search.DocIdSetIterator;
> +import org.apache.lucene.util.OpenBitSet;
> +import org.apache.lucene.util.OpenBitSetDISI;
> +import org.apache.lucene.util.SortedVIntList;
>
> /**
> * <p>
> @@ -79,29 +83,13 @@
> */
> public class ChainedFilter extends Filter
> {
> -    /**
> -     * {@link BitSet#or}.
> -     */
>    public static final int OR = 0;
> -
> -    /**
> -     * {@link BitSet#and}.
> -     */
>    public static final int AND = 1;
> -
> -    /**
> -     * {@link BitSet#andNot}.
> -     */
>    public static final int ANDNOT = 2;
> -
> -    /**
> -     * {@link BitSet#xor}.
> -     */
>    public static final int XOR = 3;
> -
>    /**
>     * Logical operation when none is declared. Defaults to
> -     * {@link BitSet#or}.
> +     * OR.
>     */
>    public static int DEFAULT = OR;
>
> @@ -144,96 +132,95 @@
>    }
>
>    /**
> -     * {@link Filter#bits}.
> +     * {@link Filter#getDocIdSet}.
>     */
> -    public BitSet bits(IndexReader reader) throws IOException
> +    public DocIdSet getDocIdSet(IndexReader reader) throws  
> IOException
>    {
> +        int[] index = new int[1]; // use array as reference to  
> modifiable int;
> +        index[0] = 0;             // an object attribute would not  
> be thread safe.
>        if (logic != -1)
> -            return bits(reader, logic);
> +            return getDocIdSet(reader, logic, index);
>        else if (logicArray != null)
> -            return bits(reader, logicArray);
> +            return getDocIdSet(reader, logicArray, index);
>        else
> -            return bits(reader, DEFAULT);
> +            return getDocIdSet(reader, DEFAULT, index);
>    }
>
> -    /**
> -     * Delegates to each filter in the chain.
> -     * @param reader IndexReader
> -     * @param logic Logical operation
> -     * @return BitSet
> -     */
> -    private BitSet bits(IndexReader reader, int logic) throws  
> IOException
> +    private DocIdSetIterator getDISI(Filter filter, IndexReader  
> reader)
> +    throws IOException
>    {
> -        BitSet result;
> -        int i = 0;
> +        return filter.getDocIdSet(reader).iterator();
> +    }
>
> +    private OpenBitSetDISI initialResult(IndexReader reader, int  
> logic, int[] index)
> +    throws IOException
> +    {
> +        OpenBitSetDISI result;
>        /**
>         * First AND operation takes place against a completely false
> -         * bitset and will always return zero results. Thanks to
> -         * Daniel Armbrust for pointing this out and suggesting  
> workaround.
> +         * bitset and will always return zero results.
>         */
>        if (logic == AND)
>        {
> -            result = (BitSet) chain[i].bits(reader).clone();
> -            ++i;
> +            result = new OpenBitSetDISI(getDISI(chain[index[0]],  
> reader), reader.maxDoc());
> +            ++index[0];
>        }
>        else if (logic == ANDNOT)
>        {
> -            result = (BitSet) chain[i].bits(reader).clone();
> -            result.flip(0,reader.maxDoc());
> -            ++i;
> +            result = new OpenBitSetDISI(getDISI(chain[index[0]],  
> reader), reader.maxDoc());
> +            result.flip(0,reader.maxDoc()); // NOTE: may set bits  
> for deleted docs.
> +            ++index[0];
>        }
>        else
>        {
> -            result = new BitSet(reader.maxDoc());
> +            result = new OpenBitSetDISI(reader.maxDoc());
>        }
> +        return result;
> +    }
> +
> +    /** Provide a SortedVIntList when it is definitely smaller than  
> an OpenBitSet */
> +    protected DocIdSet finalResult(OpenBitSetDISI result, int  
> maxDocs) {
> +        return (result.cardinality() < (maxDocs / 9))
> +              ? (DocIdSet) new SortedVIntList(result)
> +              : (DocIdSet) result;
> +    }
> +
>
> -        for (; i < chain.length; i++)
> +    /**
> +     * Delegates to each filter in the chain.
> +     * @param reader IndexReader
> +     * @param logic Logical operation
> +     * @return DocIdSet
> +     */
> +    private DocIdSet getDocIdSet(IndexReader reader, int logic,  
> int[] index)
> +    throws IOException
> +    {
> +        OpenBitSetDISI result = initialResult(reader, logic, index);
> +        for (; index[0] < chain.length; index[0]++)
>        {
> -            doChain(result, reader, logic, chain[i]);
> +            doChain(result, logic,  
> chain[index[0]].getDocIdSet(reader));
>        }
> -        return result;
> +        return finalResult(result, reader.maxDoc());
>    }
>
>    /**
>     * Delegates to each filter in the chain.
>     * @param reader IndexReader
>     * @param logic Logical operation
> -     * @return BitSet
> +     * @return DocIdSet
>     */
> -    private BitSet bits(IndexReader reader, int[] logic) throws  
> IOException
> +    private DocIdSet getDocIdSet(IndexReader reader, int[] logic,  
> int[] index)
> +    throws IOException
>    {
>        if (logic.length != chain.length)
>            throw new IllegalArgumentException("Invalid number of  
> elements in logic array");
> -        BitSet result;
> -        int i = 0;
> -
> -        /**
> -         * First AND operation takes place against a completely false
> -         * bitset and will always return zero results. Thanks to
> -         * Daniel Armbrust for pointing this out and suggesting  
> workaround.
> -         */
> -        if (logic[0] == AND)
> -        {
> -            result = (BitSet) chain[i].bits(reader).clone();
> -            ++i;
> -        }
> -        else if (logic[0] == ANDNOT)
> -        {
> -            result = (BitSet) chain[i].bits(reader).clone();
> -            result.flip(0,reader.maxDoc());
> -            ++i;
> -        }
> -        else
> -        {
> -            result = new BitSet(reader.maxDoc());
> -        }
>
> -        for (; i < chain.length; i++)
> +        OpenBitSetDISI result = initialResult(reader, logic[0],  
> index);
> +        for (; index[0] < chain.length; index[0]++)
>        {
> -            doChain(result, reader, logic[i], chain[i]);
> +            doChain(result, logic[index[0]],  
> chain[index[0]].getDocIdSet(reader));
>        }
> -        return result;
> +        return finalResult(result, reader.maxDoc());
>    }
>
>    public String toString()
> @@ -249,26 +236,51 @@
>        return sb.toString();
>    }
>
> -    private void doChain(BitSet result, IndexReader reader,
> -                         int logic, Filter filter) throws IOException
> +    private void doChain(OpenBitSetDISI result, int logic, DocIdSet  
> dis)
> +    throws IOException
>    {
> +
> +      if (dis instanceof OpenBitSet) {
> +        // optimized case for OpenBitSets
>        switch (logic)
>        {
>            case OR:
> -                result.or(filter.bits(reader));
> +                result.or((OpenBitSet) dis);
>                break;
>            case AND:
> -                result.and(filter.bits(reader));
> +                result.and((OpenBitSet) dis);
>                break;
>            case ANDNOT:
> -                result.andNot(filter.bits(reader));
> +                result.andNot((OpenBitSet) dis);
>                break;
>            case XOR:
> -                result.xor(filter.bits(reader));
> +                result.xor((OpenBitSet) dis);
>                break;
>            default:
> -                doChain(result, reader, DEFAULT, filter);
> +                doChain(result, DEFAULT, dis);
>                break;
>        }
> +      } else {
> +        DocIdSetIterator disi = dis.iterator();
> +        switch (logic)
> +        {
> +            case OR:
> +                result.inPlaceOr(disi);
> +                break;
> +            case AND:
> +                result.inPlaceAnd(disi);
> +                break;
> +            case ANDNOT:
> +                result.inPlaceNot(disi);
> +                break;
> +            case XOR:
> +                result.inPlaceXor(disi);
> +                break;
> +            default:
> +                doChain(result, DEFAULT, dis);
> +                break;
> +        }
> +      }
>    }
> +
> }
>
> Modified: lucene/java/trunk/contrib/miscellaneous/src/test/org/ 
> apache/lucene/misc/ChainedFilterTest.java
> URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/miscellaneous/src/test/org/apache/lucene/misc/ChainedFilterTest.java?rev=659635&r1=659634&r2=659635&view=diff
> = 
> = 
> = 
> = 
> = 
> = 
> = 
> = 
> ======================================================================
> --- lucene/java/trunk/contrib/miscellaneous/src/test/org/apache/ 
> lucene/misc/ChainedFilterTest.java (original)
> +++ lucene/java/trunk/contrib/miscellaneous/src/test/org/apache/ 
> lucene/misc/ChainedFilterTest.java Fri May 23 12:25:05 2008
> @@ -19,11 +19,17 @@
>
> import junit.framework.TestCase;
> import java.util.*;
> +import java.io.IOException;
> import java.text.ParseException;
> import java.text.SimpleDateFormat;
> import org.apache.lucene.index.IndexWriter;
> import org.apache.lucene.index.Term;
> +import org.apache.lucene.index.IndexWriter.MaxFieldLength;
> +import org.apache.lucene.store.Directory;
> +import org.apache.lucene.store.FSDirectory;
> +import org.apache.lucene.store.NoLockFactory;
> import org.apache.lucene.store.RAMDirectory;
> +import org.apache.lucene.analysis.Analyzer;
> import org.apache.lucene.analysis.WhitespaceAnalyzer;
> import org.apache.lucene.document.Document;
> import org.apache.lucene.document.Field;
> @@ -80,76 +86,149 @@
>        new TermQuery(new Term("owner", "sue")));
>  }
>
> -  public void testSingleFilter() throws Exception {
> -    ChainedFilter chain = new ChainedFilter(
> -        new Filter[] {dateFilter});
> -
> -    Hits hits = searcher.search(query, chain);
> -    assertEquals(MAX, hits.length());
> -
> -    chain = new ChainedFilter(new Filter[] {bobFilter});
> -    hits = searcher.search(query, chain);
> -    assertEquals(MAX / 2, hits.length());
> +  private Filter[] getChainWithOldFilters(Filter[] chain) {
> +    Filter[] oldFilters = new Filter[chain.length];
> +    for (int i = 0; i < chain.length; i++) {
> +      oldFilters[i] = new OldBitSetFilterWrapper(chain[i]);
> +    }
> +    return oldFilters;
> +  }
> +
> +  private ChainedFilter getChainedFilter(Filter[] chain, int[]  
> logic, boolean old) {
> +    if (old) {
> +      chain = getChainWithOldFilters(chain);
> +    }
>
> -    chain = new ChainedFilter(new Filter[] {bobFilter}, new int[]  
> {ChainedFilter.AND});
> -    hits = searcher.search(query, chain);
> -    assertEquals(MAX / 2, hits.length());
> -    assertEquals("bob", hits.doc(0).get("owner"));
> +    if (logic == null) {
> +      return new ChainedFilter(chain);
> +    } else {
> +      return new ChainedFilter(chain, logic);
> +    }
> +  }
> +
> +  private ChainedFilter getChainedFilter(Filter[] chain, int logic,  
> boolean old) {
> +    if (old) {
> +      chain = getChainWithOldFilters(chain);
> +    }
>
> -    chain = new ChainedFilter(new Filter[] {bobFilter}, new int[]  
> {ChainedFilter.ANDNOT});
> -    hits = searcher.search(query, chain);
> -    assertEquals(MAX / 2, hits.length());
> -    assertEquals("sue", hits.doc(0).get("owner"));
> +    return new ChainedFilter(chain, logic);
>  }
>
> -  public void testOR() throws Exception {
> -    ChainedFilter chain = new ChainedFilter(
> -      new Filter[] {sueFilter, bobFilter});
> +
> +  public void testSingleFilter() throws Exception {
> +    for (int mode = 0; mode < 2; mode++) {
> +      boolean old = (mode==0);
> +
> +      ChainedFilter chain = getChainedFilter(new Filter[]  
> {dateFilter}, null, old);
> +
> +      Hits hits = searcher.search(query, chain);
> +      assertEquals(MAX, hits.length());
> +
> +      chain = new ChainedFilter(new Filter[] {bobFilter});
> +      hits = searcher.search(query, chain);
> +      assertEquals(MAX / 2, hits.length());
> +
> +      chain = getChainedFilter(new Filter[] {bobFilter}, new int[]  
> {ChainedFilter.AND}, old);
> +      hits = searcher.search(query, chain);
> +      assertEquals(MAX / 2, hits.length());
> +      assertEquals("bob", hits.doc(0).get("owner"));
> +
> +      chain = getChainedFilter(new Filter[] {bobFilter}, new int[]  
> {ChainedFilter.ANDNOT}, old);
> +      hits = searcher.search(query, chain);
> +      assertEquals(MAX / 2, hits.length());
> +      assertEquals("sue", hits.doc(0).get("owner"));
> +    }
> +  }
>
> -    Hits hits = searcher.search(query, chain);
> -    assertEquals("OR matches all", MAX, hits.length());
> +  public void testOR() throws Exception {
> +    for (int mode = 0; mode < 2; mode++) {
> +      boolean old = (mode==0);
> +      ChainedFilter chain = getChainedFilter(
> +        new Filter[] {sueFilter, bobFilter}, null, old);
> +
> +      Hits hits = searcher.search(query, chain);
> +      assertEquals("OR matches all", MAX, hits.length());
> +    }
>  }
>
>  public void testAND() throws Exception {
> -    ChainedFilter chain = new ChainedFilter(
> -      new Filter[] {dateFilter, bobFilter}, ChainedFilter.AND);
> -
> -    Hits hits = searcher.search(query, chain);
> -    assertEquals("AND matches just bob", MAX / 2, hits.length());
> -    assertEquals("bob", hits.doc(0).get("owner"));
> +    for (int mode = 0; mode < 2; mode++) {
> +      boolean old = (mode==0);
> +      ChainedFilter chain = getChainedFilter(
> +        new Filter[] {dateFilter, bobFilter}, ChainedFilter.AND,  
> old);
> +
> +      Hits hits = searcher.search(query, chain);
> +      assertEquals("AND matches just bob", MAX / 2, hits.length());
> +      assertEquals("bob", hits.doc(0).get("owner"));
> +    }
>  }
>
>  public void testXOR() throws Exception {
> -    ChainedFilter chain = new ChainedFilter(
> -      new Filter[]{dateFilter, bobFilter}, ChainedFilter.XOR);
> -
> -    Hits hits = searcher.search(query, chain);
> -    assertEquals("XOR matches sue", MAX / 2, hits.length());
> -    assertEquals("sue", hits.doc(0).get("owner"));
> +    for (int mode = 0; mode < 2; mode++) {
> +      boolean old = (mode==0);
> +      ChainedFilter chain = getChainedFilter(
> +        new Filter[]{dateFilter, bobFilter}, ChainedFilter.XOR, old);
> +
> +      Hits hits = searcher.search(query, chain);
> +      assertEquals("XOR matches sue", MAX / 2, hits.length());
> +      assertEquals("sue", hits.doc(0).get("owner"));
> +    }
>  }
>
>  public void testANDNOT() throws Exception {
> -    ChainedFilter chain = new ChainedFilter(
> -      new Filter[]{dateFilter, sueFilter},
> -        new int[] {ChainedFilter.AND, ChainedFilter.ANDNOT});
> -
> -    Hits hits = searcher.search(query, chain);
> -    assertEquals("ANDNOT matches just bob",
> -        MAX / 2, hits.length());
> -    assertEquals("bob", hits.doc(0).get("owner"));
> -
> -    chain = new ChainedFilter(
> -        new Filter[]{bobFilter, bobFilter},
> -          new int[] {ChainedFilter.ANDNOT, ChainedFilter.ANDNOT});
> -
> -      hits = searcher.search(query, chain);
> -      assertEquals("ANDNOT bob ANDNOT bob matches all sues",
> +    for (int mode = 0; mode < 2; mode++) {
> +      boolean old = (mode==0);
> +      ChainedFilter chain = getChainedFilter(
> +        new Filter[]{dateFilter, sueFilter},
> +          new int[] {ChainedFilter.AND, ChainedFilter.ANDNOT}, old);
> +
> +      Hits hits = searcher.search(query, chain);
> +      assertEquals("ANDNOT matches just bob",
>          MAX / 2, hits.length());
> -      assertEquals("sue", hits.doc(0).get("owner"));
> +      assertEquals("bob", hits.doc(0).get("owner"));
> +
> +      chain = getChainedFilter(
> +          new Filter[]{bobFilter, bobFilter},
> +            new int[] {ChainedFilter.ANDNOT, ChainedFilter.ANDNOT},  
> old);
> +
> +        hits = searcher.search(query, chain);
> +        assertEquals("ANDNOT bob ANDNOT bob matches all sues",
> +            MAX / 2, hits.length());
> +        assertEquals("sue", hits.doc(0).get("owner"));
> +    }
>  }
>
>  private Date parseDate(String s) throws ParseException {
>    return new SimpleDateFormat("yyyy MMM dd", Locale.US).parse(s);
>  }
> +
> +  public void testWithCachingFilter() throws Exception {
> +    for (int mode = 0; mode < 2; mode++) {
> +      boolean old = (mode==0);
> +      Directory dir = new RAMDirectory();
> +      Analyzer analyzer = new WhitespaceAnalyzer();
> +
> +      IndexWriter writer = new IndexWriter(dir, analyzer, true,  
> MaxFieldLength.LIMITED);
> +      writer.close();
> +
> +      Searcher searcher = new IndexSearcher(dir);
> +
> +      Query query = new TermQuery(new Term("none", "none"));
> +
> +      QueryWrapperFilter queryFilter = new QueryWrapperFilter(query);
> +      CachingWrapperFilter cachingFilter = new  
> CachingWrapperFilter(queryFilter);
> +
> +      searcher.search(query, cachingFilter, 1);
> +
> +      CachingWrapperFilter cachingFilter2 = new  
> CachingWrapperFilter(queryFilter);
> +      Filter[] chain = new Filter[2];
> +      chain[0] = cachingFilter;
> +      chain[1] = cachingFilter2;
> +      ChainedFilter cf = new ChainedFilter(chain);
> +
> +      // throws java.lang.ClassCastException:  
> org.apache.lucene.util.OpenBitSet cannot be cast to java.util.BitSet
> +      searcher.search(new MatchAllDocsQuery(), cf, 1);
> +    }
> +  }
>
> }
>
> Modified: lucene/java/trunk/contrib/queries/src/java/org/apache/ 
> lucene/search/BooleanFilter.java
> URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/queries/src/java/org/apache/lucene/search/BooleanFilter.java?rev=659635&r1=659634&r2=659635&view=diff
> = 
> = 
> = 
> = 
> = 
> = 
> = 
> = 
> ======================================================================
> --- lucene/java/trunk/contrib/queries/src/java/org/apache/lucene/ 
> search/BooleanFilter.java (original)
> +++ lucene/java/trunk/contrib/queries/src/java/org/apache/lucene/ 
> search/BooleanFilter.java Fri May 23 12:25:05 2008
> @@ -23,6 +23,10 @@
>
> import org.apache.lucene.index.IndexReader;
> import org.apache.lucene.search.BooleanClause.Occur;
> +import org.apache.lucene.util.DocIdBitSet;
> +import org.apache.lucene.util.OpenBitSet;
> +import org.apache.lucene.util.OpenBitSetDISI;
> +import org.apache.lucene.util.SortedVIntList;
>
> /**
> * A container Filter that allows Boolean composition of Filters.
> @@ -37,184 +41,167 @@
>
> public class BooleanFilter extends Filter
> {
> -	//ArrayList of SHOULD filters
> -	ArrayList shouldFilters = null;
> -	//ArrayList of NOT filters
> -	ArrayList notFilters = null;
> -	//ArrayList of MUST filters
> -	ArrayList mustFilters = null;
> -
> -	/**
> -	 * Returns the a BitSet representing the Boolean composition
> -	 * of the filters that have been added.
> -	 */
> -	
> -	public BitSet bits(IndexReader reader) throws IOException
> -	{
> -		//create a new bitSet
> -		BitSet returnBits = null;
> -		
> -		//SHOULD filters
> -		if (shouldFilters!=null)
> -		{
> -			returnBits = ((Filter)shouldFilters.get(0)).bits(reader);
> -//			avoid changing the original bitset - it may be cached
> -			returnBits=(BitSet) returnBits.clone();
> -			if (shouldFilters.size() > 1)
> -			{
> -				for (int i = 1; i < shouldFilters.size(); i++)
> -				{
> -					returnBits.or(((Filter)shouldFilters.get(i)).bits(reader));
> -				}
> -			}
> -		}
> -		
> -		//NOT filters
> -		if (notFilters!=null)
> -		{
> -			for (int i = 0; i < notFilters.size(); i++)
> -			{
> -				BitSet notBits=((Filter)notFilters.get(i)).bits(reader);
> -				if(returnBits==null)
> -				{
> -					returnBits=(BitSet) notBits.clone();					
> -					returnBits.flip(0,reader.maxDoc());
> -				}
> -				else
> -				{
> -					returnBits.andNot(notBits);
> -				}
> -			}
> -		}
> -		
> -		//MUST filters
> -		if (mustFilters!=null)
> -		{
> -			for (int i = 0; i < mustFilters.size(); i++)
> -			{
> -				BitSet mustBits=((Filter)mustFilters.get(i)).bits(reader);
> -				if(returnBits==null)
> -				{
> -					if(mustFilters.size()==1)
> -					{
> -						returnBits=mustBits;
> -						
> -					}
> -					else
> -					{
> -						//don't mangle the bitset
> -						returnBits=(BitSet) mustBits.clone();						
> -					}
> -				}
> -				else
> -				{
> -					returnBits.and(mustBits);
> -				}
> -			}
> -		}
> -		if(returnBits==null)
> -		{
> -			returnBits=new BitSet(reader.maxDoc());
> -		}
> -		return returnBits;
> -	}
> -	
> -	/**
> -	 * Adds a new FilterClause to the Boolean Filter container
> -	 * @param filterClause A FilterClause object containing a Filter  
> and an Occur parameter
> -	 */
> -	
> -	public void add(FilterClause filterClause)
> -	{
> -		if (filterClause.getOccur().equals(Occur.MUST))
> -		{
> -			if(mustFilters==null)
> -			{
> -				mustFilters=new ArrayList();
> -			}
> -			mustFilters.add(filterClause.getFilter());
> -		}
> -		if (filterClause.getOccur().equals(Occur.SHOULD))
> -		{
> -			if(shouldFilters==null)
> -			{
> -				shouldFilters=new ArrayList();
> -			}
> -			shouldFilters.add(filterClause.getFilter());
> -		}
> -		if (filterClause.getOccur().equals(Occur.MUST_NOT))
> -		{
> -			if(notFilters==null)
> -			{
> -				notFilters=new ArrayList();
> -			}
> -			notFilters.add(filterClause.getFilter());
> -		}
> -	}
> -
> -	public boolean equals(Object obj)
> -	{
> -		if(this == obj)
> -			return true;
> -		if((obj == null) || (obj.getClass() != this.getClass()))
> -				return false;
> -		BooleanFilter test = (BooleanFilter)obj;
> -		return (notFilters == test.notFilters||
> -					 (notFilters!= null && notFilters.equals(test.notFilters)))
> -				&&
> -			   (mustFilters == test.mustFilters||
> -					 (mustFilters!= null &&  
> mustFilters.equals(test.mustFilters)))				
> -					 &&
> -			   (shouldFilters == test.shouldFilters||
> -					 (shouldFilters!= null &&  
> shouldFilters.equals(test.shouldFilters)));
> -	}
> -
> -	public int hashCode()
> -	{
> -		int hash=7;
> -		hash = 31 * hash + (null == mustFilters ? 0 :  
> mustFilters.hashCode());
> -		hash = 31 * hash + (null == notFilters ? 0 :  
> notFilters.hashCode());
> -		hash = 31 * hash + (null == shouldFilters ? 0 :  
> shouldFilters.hashCode());
> -		return hash;
> -	}
> -	
> -	
> -		/** Prints a user-readable version of this query. */
> -	public String toString()
> -	{
> -		StringBuffer buffer = new StringBuffer();
> -
> -		buffer.append("BooleanFilter(");
> -
> -		appendFilters(shouldFilters, null, buffer);
> -		appendFilters(mustFilters, "+", buffer);
> -		appendFilters(notFilters, "-", buffer);
> -
> -		buffer.append(")");
> -
> -		return buffer.toString();
> -	}
> -	
> -	private void appendFilters(ArrayList filters, String occurString,
> -			StringBuffer buffer)
> -	{
> -		if (filters == null)
> -			return;
> -
> -		for (int i = 0; i < filters.size(); i++)
> -		{
> -			Filter filter = (Filter) filters.get(i);
> -			if (occurString != null)
> -			{
> -				buffer.append(occurString);
> -			}
> -
> -			buffer.append(filter);
> -
> -			if (i < filters.size() - 1)
> -			{
> -				buffer.append(' ');
> -			}
> -		}
> -	}		
> +  ArrayList shouldFilters = null;
> +  ArrayList notFilters = null;
> +  ArrayList mustFilters = null;
> +
> +  private DocIdSetIterator getDISI(ArrayList filters, int index,  
> IndexReader reader)
> +  throws IOException
> +  {
> +    return  
> ((Filter)filters.get(index)).getDocIdSet(reader).iterator();
> +  }
> +
> +  /**
> +   * Returns the a DocIdSetIterator representing the Boolean  
> composition
> +   * of the filters that have been added.
> +   */
> +  public DocIdSet getDocIdSet(IndexReader reader) throws IOException
> +  {
> +    OpenBitSetDISI res = null;
> +
> +    if (shouldFilters != null) {
> +      for (int i = 0; i < shouldFilters.size(); i++) {
> +        if (res == null) {
> +          res = new OpenBitSetDISI(getDISI(shouldFilters, i,  
> reader), reader.maxDoc());
> +        } else {
> +          DocIdSet dis =  
> ((Filter)shouldFilters.get(i)).getDocIdSet(reader);
> +          if(dis instanceof OpenBitSet) {
> +            // optimized case for OpenBitSets
> +            res.or((OpenBitSet) dis);
> +          } else {
> +            res.inPlaceOr(getDISI(shouldFilters, i, reader));
> +          }
> +        }
> +      }
> +    }
> +
> +    if (notFilters!=null) {
> +      for (int i = 0; i < notFilters.size(); i++) {
> +        if (res == null) {
> +          res = new OpenBitSetDISI(getDISI(notFilters, i, reader),  
> reader.maxDoc());
> +          res.flip(0, reader.maxDoc()); // NOTE: may set bits on  
> deleted docs
> +        } else {
> +          DocIdSet dis =  
> ((Filter)notFilters.get(i)).getDocIdSet(reader);
> +          if(dis instanceof OpenBitSet) {
> +            // optimized case for OpenBitSets
> +            res.andNot((OpenBitSet) dis);
> +          } else {
> +            res.inPlaceNot(getDISI(notFilters, i, reader));
> +          }
> +        }
> +      }
> +    }
> +
> +    if (mustFilters!=null) {
> +      for (int i = 0; i < mustFilters.size(); i++) {
> +        if (res == null) {
> +          res = new OpenBitSetDISI(getDISI(mustFilters, i, reader),  
> reader.maxDoc());
> +        } else {
> +          DocIdSet dis =  
> ((Filter)mustFilters.get(i)).getDocIdSet(reader);
> +          if(dis instanceof OpenBitSet) {
> +            // optimized case for OpenBitSets
> +            res.and((OpenBitSet) dis);
> +          } else {
> +            res.inPlaceAnd(getDISI(mustFilters, i, reader));
> +          }
> +        }
> +      }
> +    }
> +
> +    if (res !=null)
> +      return finalResult(res, reader.maxDoc());
> +
> +    if (emptyDocIdSet == null)
> +      emptyDocIdSet = new OpenBitSetDISI(1);
> +
> +    return emptyDocIdSet;
> +  }
> +
> +  /** Provide a SortedVIntList when it is definitely smaller than  
> an OpenBitSet */
> +  protected DocIdSet finalResult(OpenBitSetDISI result, int  
> maxDocs) {
> +    return (result.cardinality() < (maxDocs / 9))
> +      ? (DocIdSet) new SortedVIntList(result)
> +      : (DocIdSet) result;
> +  }
> +
> +  private static DocIdSet emptyDocIdSet = null;
> +
> +  /**
> +  * Adds a new FilterClause to the Boolean Filter container
> +  * @param filterClause A FilterClause object containing a Filter  
> and an Occur parameter
> +  */
> +
> +  public void add(FilterClause filterClause)
> +  {
> +    if (filterClause.getOccur().equals(Occur.MUST)) {
> +      if (mustFilters==null) {
> +        mustFilters=new ArrayList();
> +      }
> +      mustFilters.add(filterClause.getFilter());
> +    }
> +    if (filterClause.getOccur().equals(Occur.SHOULD)) {
> +      if (shouldFilters==null) {
> +        shouldFilters=new ArrayList();
> +      }
> +      shouldFilters.add(filterClause.getFilter());
> +    }
> +    if (filterClause.getOccur().equals(Occur.MUST_NOT)) {
> +      if (notFilters==null) {
> +        notFilters=new ArrayList();
> +      }
> +      notFilters.add(filterClause.getFilter());
> +    }
> +  }
> +
> +  private boolean equalFilters(ArrayList filters1, ArrayList  
> filters2)
> +  {
> +     return (filters1 == filters2) ||
> +              ((filters1 != null) && filters1.equals(filters2));
> +  }
> +
> +  public boolean equals(Object obj)
> +  {
> +    if (this == obj)
> +      return true;
> +
> +    if ((obj == null) || (obj.getClass() != this.getClass()))
> +      return false;
> +
> +    BooleanFilter other = (BooleanFilter)obj;
> +    return equalFilters(notFilters, other.notFilters)
> +        && equalFilters(mustFilters, other.mustFilters)
> +        && equalFilters(shouldFilters, other.shouldFilters);
> +  }
> +
> +  public int hashCode()
> +  {
> +    int hash=7;
> +    hash = 31 * hash + (null == mustFilters ? 0 :  
> mustFilters.hashCode());
> +    hash = 31 * hash + (null == notFilters ? 0 :  
> notFilters.hashCode());
> +    hash = 31 * hash + (null == shouldFilters ? 0 :  
> shouldFilters.hashCode());
> +    return hash;
> +  }
> +
> +  /** Prints a user-readable version of this query. */
> +  public String toString()
> +  {
> +    StringBuffer buffer = new StringBuffer();
> +    buffer.append("BooleanFilter(");
> +    appendFilters(shouldFilters, "", buffer);
> +    appendFilters(mustFilters, "+", buffer);
> +    appendFilters(notFilters, "-", buffer);
> +    buffer.append(")");
> +    return buffer.toString();
> +  }
> +
> +  private void appendFilters(ArrayList filters, String occurString,  
> StringBuffer buffer)
> +  {
> +    if (filters != null) {
> +      for (int i = 0; i < filters.size(); i++) {
> +        buffer.append(' ');
> +        buffer.append(occurString);
> +        buffer.append(filters.get(i).toString());
> +      }
> +    }
> +  }
> }
>
> Modified: lucene/java/trunk/contrib/queries/src/java/org/apache/ 
> lucene/search/DuplicateFilter.java
> URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/queries/src/java/org/apache/lucene/search/DuplicateFilter.java?rev=659635&r1=659634&r2=659635&view=diff
> = 
> = 
> = 
> = 
> = 
> = 
> = 
> = 
> ======================================================================
> --- lucene/java/trunk/contrib/queries/src/java/org/apache/lucene/ 
> search/DuplicateFilter.java (original)
> +++ lucene/java/trunk/contrib/queries/src/java/org/apache/lucene/ 
> search/DuplicateFilter.java Fri May 23 12:25:05 2008
> @@ -22,6 +22,7 @@
> import org.apache.lucene.index.Term;
> import org.apache.lucene.index.TermDocs;
> import org.apache.lucene.index.TermEnum;
> +import org.apache.lucene.util.OpenBitSet;
>
> public class DuplicateFilter extends Filter
> {
> @@ -66,7 +67,7 @@
> 		this.processingMode = processingMode;
> 	}
>
> -	public BitSet bits(IndexReader reader) throws IOException
> +  public DocIdSet getDocIdSet(IndexReader reader) throws IOException
> 	{
> 		if(processingMode==PM_FAST_INVALIDATION)
> 		{
> @@ -78,10 +79,10 @@
> 		}
> 	}
> 	
> -	private BitSet correctBits(IndexReader reader) throws IOException
> +  private OpenBitSet correctBits(IndexReader reader) throws  
> IOException
> 	{
> 		
> -		BitSet bits=new BitSet(reader.maxDoc()); //assume all are INvalid
> +    OpenBitSet bits=new OpenBitSet(reader.maxDoc()); //assume all  
> are INvalid
> 		Term startTerm=new Term(fieldName,"");
> 		TermEnum te = reader.terms(startTerm);
> 		if(te!=null)
> @@ -117,10 +118,10 @@
> 		return bits;
> 	}
> 	
> -	private BitSet fastBits(IndexReader reader) throws IOException
> +  private OpenBitSet fastBits(IndexReader reader) throws IOException
> 	{
> 		
> -		BitSet bits=new BitSet(reader.maxDoc());
> +    OpenBitSet bits=new OpenBitSet(reader.maxDoc());
> 		bits.set(0,reader.maxDoc()); //assume all are valid
> 		Term startTerm=new Term(fieldName,"");
> 		TermEnum te = reader.terms(startTerm);
> @@ -143,7 +144,7 @@
> 					do
> 					{
> 						lastDoc=td.doc();
> -						bits.set(lastDoc,false);
> +            bits.clear(lastDoc);
> 					}while(td.next());
> 					if(keepMode==KM_USE_LAST_OCCURRENCE)
> 					{
>
> Modified: lucene/java/trunk/contrib/queries/src/java/org/apache/ 
> lucene/search/TermsFilter.java
> URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/queries/src/java/org/apache/lucene/search/TermsFilter.java?rev=659635&r1=659634&r2=659635&view=diff
> = 
> = 
> = 
> = 
> = 
> = 
> = 
> = 
> ======================================================================
> --- lucene/java/trunk/contrib/queries/src/java/org/apache/lucene/ 
> search/TermsFilter.java (original)
> +++ lucene/java/trunk/contrib/queries/src/java/org/apache/lucene/ 
> search/TermsFilter.java Fri May 23 12:25:05 2008
> @@ -26,6 +26,7 @@
> import org.apache.lucene.index.IndexReader;
> import org.apache.lucene.index.Term;
> import org.apache.lucene.index.TermDocs;
> +import org.apache.lucene.util.OpenBitSet;
>
> /**
> * Constructs a filter for docs matching any of the terms added to  
> this class.
> @@ -50,11 +51,11 @@
> 	}
>
> 	/* (non-Javadoc)
> -	 * @see  
> org 
> .apache.lucene.search.Filter#bits(org.apache.lucene.index.IndexReader)
> +   * @see  
> org 
> .apache 
> .lucene.search.Filter#getDocIdSet(org.apache.lucene.index.IndexReader)
> 	 */
> -	public BitSet bits(IndexReader reader) throws IOException
> +  public DocIdSet getDocIdSet(IndexReader reader) throws IOException
> 	{
> -		BitSet result=new BitSet(reader.maxDoc());
> +    OpenBitSet result=new OpenBitSet(reader.maxDoc());
>        TermDocs td = reader.termDocs();
>        try
>        {
>
> Modified: lucene/java/trunk/contrib/queries/src/test/org/apache/ 
> lucene/search/BooleanFilterTest.java
> URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/queries/src/test/org/apache/lucene/search/BooleanFilterTest.java?rev=659635&r1=659634&r2=659635&view=diff
> = 
> = 
> = 
> = 
> = 
> = 
> = 
> = 
> ======================================================================
> --- lucene/java/trunk/contrib/queries/src/test/org/apache/lucene/ 
> search/BooleanFilterTest.java (original)
> +++ lucene/java/trunk/contrib/queries/src/test/org/apache/lucene/ 
> search/BooleanFilterTest.java Fri May 23 12:25:05 2008
> @@ -32,6 +32,7 @@
> import org.apache.lucene.search.FilterClause;
> import org.apache.lucene.search.RangeFilter;
> import org.apache.lucene.store.RAMDirectory;
> +import org.apache.lucene.util.DocIdBitSet;
>
> import junit.framework.TestCase;
>
> @@ -66,100 +67,141 @@
> 		writer.addDocument(doc);
> 	}
> 	
> -	private Filter getRangeFilter(String field,String lowerPrice,  
> String upperPrice)
> +  private Filter getRangeFilter(String field,String lowerPrice,  
> String upperPrice, boolean old)
> 	{
> -		return new RangeFilter(field,lowerPrice,upperPrice,true,true);
> +    Filter f = new  
> RangeFilter(field,lowerPrice,upperPrice,true,true);
> +    if (old) {
> +      return new OldBitSetFilterWrapper(f);
> +    }
> +
> +    return f;
> 	}
> -	private TermsFilter getTermsFilter(String field,String text)
> +  private Filter getTermsFilter(String field,String text, boolean  
> old)
> 	{
> 		TermsFilter tf=new TermsFilter();
> 		tf.addTerm(new Term(field,text));
> +    if (old) {
> +      return new OldBitSetFilterWrapper(tf);
> +    }
> +
> 		return tf;
> 	}
> +
> +        private void tstFilterCard(String mes, int expected, Filter  
> filt)
> +        throws Throwable
> +        {
> +          DocIdSetIterator disi =  
> filt.getDocIdSet(reader).iterator();
> +          int actual = 0;
> +          while (disi.next()) {
> +            actual++;
> +          }
> +          assertEquals(mes, expected, actual);
> +        }
> +
> 		
> 	public void testShould() throws Throwable
> 	{
> -		BooleanFilter booleanFilter = new BooleanFilter();
> -		booleanFilter.add(new  
> FilterClause 
> (getTermsFilter("price","030"),BooleanClause.Occur.SHOULD));
> -		BitSet bits = booleanFilter.bits(reader);
> -		assertEquals("Should retrieves only 1 doc",1,bits.cardinality());
> +    for (int i = 0; i < 2; i++) {
> +      boolean old = (i==0);
> +      BooleanFilter booleanFilter = new BooleanFilter();
> +      booleanFilter.add(new  
> FilterClause(getTermsFilter("price","030",  
> old),BooleanClause.Occur.SHOULD));
> +      tstFilterCard("Should retrieves only 1 doc",1,booleanFilter);
> +    }
> 	}
> 	
> 	public void testShoulds() throws Throwable
> 	{
> -		BooleanFilter booleanFilter = new BooleanFilter();
> -		booleanFilter.add(new FilterClause(getRangeFilter("price","010",  
> "020"),BooleanClause.Occur.SHOULD));
> -		booleanFilter.add(new FilterClause(getRangeFilter("price","020",  
> "030"),BooleanClause.Occur.SHOULD));
> -		BitSet bits = booleanFilter.bits(reader);
> -		assertEquals("Shoulds are Ored together",5,bits.cardinality());
> +    for (int i = 0; i < 2; i++) {
> +      boolean old = (i==0);
> +      BooleanFilter booleanFilter = new BooleanFilter();
> +      booleanFilter.add(new  
> FilterClause(getRangeFilter("price","010", "020",  
> old),BooleanClause.Occur.SHOULD));
> +      booleanFilter.add(new  
> FilterClause(getRangeFilter("price","020", "030",  
> old),BooleanClause.Occur.SHOULD));
> +      tstFilterCard("Shoulds are Ored together",5,booleanFilter);
> +    }
> 	}
> 	public void testShouldsAndMustNot() throws Throwable
> 	{
> -		BooleanFilter booleanFilter = new BooleanFilter();
> -		booleanFilter.add(new FilterClause(getRangeFilter("price","010",  
> "020"),BooleanClause.Occur.SHOULD));
> -		booleanFilter.add(new FilterClause(getRangeFilter("price","020",  
> "030"),BooleanClause.Occur.SHOULD));
> -		booleanFilter.add(new FilterClause(getTermsFilter("inStock",  
> "N"),BooleanClause.Occur.MUST_NOT));
> -		BitSet bits = booleanFilter.bits(reader);
> -		assertEquals("Shoulds Ored but AndNot",4,bits.cardinality());
> -
> -		booleanFilter.add(new FilterClause(getTermsFilter("inStock",  
> "Maybe"),BooleanClause.Occur.MUST_NOT));
> -		bits = booleanFilter.bits(reader);
> -		assertEquals("Shoulds Ored but AndNots",3,bits.cardinality());
> +    for (int i = 0; i < 2; i++) {
> +      boolean old = (i==0);
> +
> +      BooleanFilter booleanFilter = new BooleanFilter();
> +      booleanFilter.add(new  
> FilterClause(getRangeFilter("price","010", "020",  
> old),BooleanClause.Occur.SHOULD));
> +      booleanFilter.add(new  
> FilterClause(getRangeFilter("price","020", "030",  
> old),BooleanClause.Occur.SHOULD));
> +      booleanFilter.add(new FilterClause(getTermsFilter("inStock",  
> "N", old),BooleanClause.Occur.MUST_NOT));
> +      tstFilterCard("Shoulds Ored but AndNot",4,booleanFilter);
> +
> +      booleanFilter.add(new FilterClause(getTermsFilter("inStock",  
> "Maybe", old),BooleanClause.Occur.MUST_NOT));
> +      tstFilterCard("Shoulds Ored but AndNots",3,booleanFilter);
> +    }
> 		
> 	}
> 	public void testShouldsAndMust() throws Throwable
> 	{
> -		BooleanFilter booleanFilter = new BooleanFilter();
> -		booleanFilter.add(new FilterClause(getRangeFilter("price","010",  
> "020"),BooleanClause.Occur.SHOULD));
> -		booleanFilter.add(new FilterClause(getRangeFilter("price","020",  
> "030"),BooleanClause.Occur.SHOULD));
> -		booleanFilter.add(new FilterClause(getTermsFilter("accessRights",  
> "admin"),BooleanClause.Occur.MUST));
> -		BitSet bits = booleanFilter.bits(reader);
> -		assertEquals("Shoulds Ored but MUST",3,bits.cardinality());
> +    for (int i = 0; i < 2; i++) {
> +      boolean old = (i==0);
> +      BooleanFilter booleanFilter = new BooleanFilter();
> +      booleanFilter.add(new  
> FilterClause(getRangeFilter("price","010", "020",  
> old),BooleanClause.Occur.SHOULD));
> +      booleanFilter.add(new  
> FilterClause(getRangeFilter("price","020", "030",  
> old),BooleanClause.Occur.SHOULD));
> +      booleanFilter.add(new  
> FilterClause(getTermsFilter("accessRights", "admin",  
> old),BooleanClause.Occur.MUST));
> +      tstFilterCard("Shoulds Ored but MUST",3,booleanFilter);
> +    }
> 	}
> 	public void testShouldsAndMusts() throws Throwable
> 	{
> -		BooleanFilter booleanFilter = new BooleanFilter();
> -		booleanFilter.add(new FilterClause(getRangeFilter("price","010",  
> "020"),BooleanClause.Occur.SHOULD));
> -		booleanFilter.add(new FilterClause(getRangeFilter("price","020",  
> "030"),BooleanClause.Occur.SHOULD));
> -		booleanFilter.add(new FilterClause(getTermsFilter("accessRights",  
> "admin"),BooleanClause.Occur.MUST));
> -		booleanFilter.add(new  
> FilterClause(getRangeFilter("date","20040101",  
> "20041231"),BooleanClause.Occur.MUST));
> -		BitSet bits = booleanFilter.bits(reader);
> -		assertEquals("Shoulds Ored but MUSTs ANDED",1,bits.cardinality());
> +    for (int i = 0; i < 2; i++) {
> +      boolean old = (i==0);
> +
> +      BooleanFilter booleanFilter = new BooleanFilter();
> +      booleanFilter.add(new  
> FilterClause(getRangeFilter("price","010", "020",  
> old),BooleanClause.Occur.SHOULD));
> +      booleanFilter.add(new  
> FilterClause(getRangeFilter("price","020", "030",  
> old),BooleanClause.Occur.SHOULD));
> +      booleanFilter.add(new  
> FilterClause(getTermsFilter("accessRights", "admin",  
> old),BooleanClause.Occur.MUST));
> +      booleanFilter.add(new  
> FilterClause(getRangeFilter("date","20040101", "20041231",  
> old),BooleanClause.Occur.MUST));
> +      tstFilterCard("Shoulds Ored but MUSTs ANDED",1,booleanFilter);
> +    }
> 	}
> 	public void testShouldsAndMustsAndMustNot() throws Throwable
> 	{
> -		BooleanFilter booleanFilter = new BooleanFilter();
> -		booleanFilter.add(new FilterClause(getRangeFilter("price","030",  
> "040"),BooleanClause.Occur.SHOULD));
> -		booleanFilter.add(new FilterClause(getTermsFilter("accessRights",  
> "admin"),BooleanClause.Occur.MUST));
> -		booleanFilter.add(new  
> FilterClause(getRangeFilter("date","20050101",  
> "20051231"),BooleanClause.Occur.MUST));
> -		booleanFilter.add(new  
> FilterClause 
> (getTermsFilter("inStock","N"),BooleanClause.Occur.MUST_NOT));
> -		BitSet bits = booleanFilter.bits(reader);
> -		assertEquals("Shoulds Ored but MUSTs ANDED and MustNot", 
> 0,bits.cardinality());
> +    for (int i = 0; i < 2; i++) {
> +      boolean old = (i==0);
> +
> +      BooleanFilter booleanFilter = new BooleanFilter();
> +      booleanFilter.add(new  
> FilterClause(getRangeFilter("price","030", "040",  
> old),BooleanClause.Occur.SHOULD));
> +      booleanFilter.add(new  
> FilterClause(getTermsFilter("accessRights", "admin",  
> old),BooleanClause.Occur.MUST));
> +      booleanFilter.add(new  
> FilterClause(getRangeFilter("date","20050101", "20051231",  
> old),BooleanClause.Occur.MUST));
> +      booleanFilter.add(new  
> FilterClause(getTermsFilter("inStock","N",  
> old),BooleanClause.Occur.MUST_NOT));
> +      tstFilterCard("Shoulds Ored but MUSTs ANDED and MustNot", 
> 0,booleanFilter);
> +    }
> 	}
> 	
> 	public void testJustMust() throws Throwable
> 	{
> -		BooleanFilter booleanFilter = new BooleanFilter();
> -		booleanFilter.add(new FilterClause(getTermsFilter("accessRights",  
> "admin"),BooleanClause.Occur.MUST));
> -		BitSet bits = booleanFilter.bits(reader);
> -		assertEquals("MUST",3,bits.cardinality());
> +    for (int i = 0; i < 2; i++) {
> +      boolean old = (i==0);
> +
> +      BooleanFilter booleanFilter = new BooleanFilter();
> +      booleanFilter.add(new  
> FilterClause(getTermsFilter("accessRights", "admin",  
> old),BooleanClause.Occur.MUST));
> +      tstFilterCard("MUST",3,booleanFilter);
> +    }
> 	}
> 	public void testJustMustNot() throws Throwable
> 	{
> -		BooleanFilter booleanFilter = new BooleanFilter();
> -		booleanFilter.add(new  
> FilterClause 
> (getTermsFilter("inStock","N"),BooleanClause.Occur.MUST_NOT));
> -		BitSet bits = booleanFilter.bits(reader);
> -		assertEquals("MUST_NOT",4,bits.cardinality());
> +    for (int i = 0; i < 2; i++) {
> +      boolean old = (i==0);
> +
> +      BooleanFilter booleanFilter = new BooleanFilter();
> +      booleanFilter.add(new  
> FilterClause(getTermsFilter("inStock","N",  
> old),BooleanClause.Occur.MUST_NOT));
> +      tstFilterCard("MUST_NOT",4,booleanFilter);
> +    }
> 	}
> 	public void testMustAndMustNot() throws Throwable
> 	{
> -		BooleanFilter booleanFilter = new BooleanFilter();
> -		booleanFilter.add(new  
> FilterClause(getTermsFilter("inStock","N"),BooleanClause.Occur.MUST));
> -		booleanFilter.add(new  
> FilterClause 
> (getTermsFilter("price","030"),BooleanClause.Occur.MUST_NOT));
> -		BitSet bits = booleanFilter.bits(reader);
> -		assertEquals("MUST_NOT wins over MUST for same docs", 
> 0,bits.cardinality());
> -	}
> +    for (int i = 0; i < 2; i++) {
> +      boolean old = (i==0);
>
> -	
> -	
> +      BooleanFilter booleanFilter = new BooleanFilter();
> +      booleanFilter.add(new  
> FilterClause(getTermsFilter("inStock","N",  
> old),BooleanClause.Occur.MUST));
> +      booleanFilter.add(new  
> FilterClause(getTermsFilter("price","030",  
> old),BooleanClause.Occur.MUST_NOT));
> +      tstFilterCard("MUST_NOT wins over MUST for same docs", 
> 0,booleanFilter);
> +    }
> +	}
> }
>
> Modified: lucene/java/trunk/src/java/org/apache/lucene/search/ 
> ParallelMultiSearcher.java
> URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/search/ParallelMultiSearcher.java?rev=659635&r1=659634&r2=659635&view=diff
> = 
> = 
> = 
> = 
> = 
> = 
> = 
> = 
> ======================================================================
> --- lucene/java/trunk/src/java/org/apache/lucene/search/ 
> ParallelMultiSearcher.java (original)
> +++ lucene/java/trunk/src/java/org/apache/lucene/search/ 
> ParallelMultiSearcher.java Fri May 23 12:25:05 2008
> @@ -150,8 +150,8 @@
>
>  /** Lower-level search API.
>   *
> -   * <p>{@link HitCollector#collect(int,float)} is called for every  
> non-zero
> -   * scoring document.
> +   * <p>{@link HitCollector#collect(int,float)} is called for every  
> matching
> +   * document.
>   *
>   * <p>Applications should only use this if they need <i>all</i> of  
> the
>   * matching documents.  The high-level search API ({@link
>
> Modified: lucene/java/trunk/src/java/org/apache/lucene/search/ 
> Searchable.java
> URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/search/Searchable.java?rev=659635&r1=659634&r2=659635&view=diff
> = 
> = 
> = 
> = 
> = 
> = 
> = 
> = 
> ======================================================================
> --- lucene/java/trunk/src/java/org/apache/lucene/search/ 
> Searchable.java (original)
> +++ lucene/java/trunk/src/java/org/apache/lucene/search/ 
> Searchable.java Fri May 23 12:25:05 2008
> @@ -33,7 +33,7 @@
> *
> * <p>Queries, filters and sort criteria are designed to be compact  
> so that
> * they may be efficiently passed to a remote index, with only the  
> top-scoring
> - * hits being returned, rather than every non-zero scoring hit.
> + * hits being returned, rather than every matching hit.
> */
> public interface Searchable extends java.rmi.Remote {
>  /** Lower-level search API.
>
> Modified: lucene/java/trunk/src/java/org/apache/lucene/search/ 
> Searcher.java
> URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/search/Searcher.java?rev=659635&r1=659634&r2=659635&view=diff
> = 
> = 
> = 
> = 
> = 
> = 
> = 
> = 
> ======================================================================
> --- lucene/java/trunk/src/java/org/apache/lucene/search/ 
> Searcher.java (original)
> +++ lucene/java/trunk/src/java/org/apache/lucene/search/ 
> Searcher.java Fri May 23 12:25:05 2008
> @@ -88,8 +88,8 @@
>
>  /** Lower-level search API.
>   *
> -   * <p>{@link HitCollector#collect(int,float)} is called for every  
> non-zero
> -   * scoring document.
> +   * <p>{@link HitCollector#collect(int,float)} is called for every  
> matching
> +   * document.
>   *
>   * <p>Applications should only use this if they need <i>all</i> of  
> the
>   * matching documents.  The high-level search API ({@link
> @@ -107,8 +107,8 @@
>
>  /** Lower-level search API.
>   *
> -   * <p>{@link HitCollector#collect(int,float)} is called for every  
> non-zero
> -   * scoring document.
> +   * <p>{@link HitCollector#collect(int,float)} is called for every  
> matching
> +   * document.
>   * <br>HitCollector-based access to remote indexes is discouraged.
>   *
>   * <p>Applications should only use this if they need <i>all</i> of  
> the
>
> Added: lucene/java/trunk/src/java/org/apache/lucene/util/ 
> OpenBitSetDISI.java
> URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/util/OpenBitSetDISI.java?rev=659635&view=auto
> = 
> = 
> = 
> = 
> = 
> = 
> = 
> = 
> ======================================================================
> --- lucene/java/trunk/src/java/org/apache/lucene/util/ 
> OpenBitSetDISI.java (added)
> +++ lucene/java/trunk/src/java/org/apache/lucene/util/ 
> OpenBitSetDISI.java Fri May 23 12:25:05 2008
> @@ -0,0 +1,101 @@
> +package org.apache.lucene.util;
> +
> +/**
> + * Licensed to the Apache Software Foundation (ASF) under one or more
> + * contributor license agreements.  See the NOTICE file distributed  
> with
> + * this work for additional information regarding copyright  
> ownership.
> + * The ASF licenses this file to You under the Apache License,  
> Version 2.0
> + * (the "License"); you may not use this file except in compliance  
> with
> + * the License.  You may obtain a copy of the License at
> + *
> + *     http://www.apache.org/licenses/LICENSE-2.0
> + *
> + * Unless required by applicable law or agreed to in writing,  
> software
> + * distributed under the License is distributed on an "AS IS" BASIS,
> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or  
> implied.
> + * See the License for the specific language governing permissions  
> and
> + * limitations under the License.
> + */
> +
> +import java.io.IOException;
> +import org.apache.lucene.search.DocIdSetIterator;
> +
> +public class OpenBitSetDISI extends OpenBitSet {
> +
> +  /** Construct an OpenBitSetDISI with its bits set
> +   * from the doc ids of the given DocIdSetIterator.
> +   * Also give a maximum size one larger than the largest doc id  
> for which a
> +   * bit may ever be set on this OpenBitSetDISI.
> +   */
> +  public OpenBitSetDISI(DocIdSetIterator disi, int maxSize) throws  
> IOException {
> +    super(maxSize);
> +    inPlaceOr(disi);
> +  }
> +
> +  /** Construct an OpenBitSetDISI with no bits set, and a given  
> maximum size
> +   * one larger than the largest doc id for which a bit may ever be  
> set
> +   * on this OpenBitSetDISI.
> +   */
> +  public OpenBitSetDISI(int maxSize) {
> +    super(maxSize);
> +  }
> +
> +  /**
> +   * Perform an inplace OR with the doc ids from a given  
> DocIdSetIterator,
> +   * setting the bit for each such doc id.
> +   * These doc ids should be smaller than the maximum size passed  
> to the
> +   * constructor.
> +   */
> +  public void inPlaceOr(DocIdSetIterator disi) throws IOException {
> +    while (disi.next() && (disi.doc() < size())) {
> +      fastSet(disi.doc());
> +    }
> +  }
> +
> +  /**
> +   * Perform an inplace AND with the doc ids from a given  
> DocIdSetIterator,
> +   * leaving only the bits set for which the doc ids are in common.
> +   * These doc ids should be smaller than the maximum size passed  
> to the
> +   * constructor.
> +   */
> +  public void inPlaceAnd(DocIdSetIterator disi) throws IOException {
> +    int index = nextSetBit(0);
> +    int lastNotCleared = -1;
> +    while ((index != -1) && disi.skipTo(index)) {
> +      while ((index != -1) && (index < disi.doc())) {
> +        fastClear(index);
> +        index = nextSetBit(index + 1);
> +      }
> +      if (index == disi.doc()) {
> +        lastNotCleared = index;
> +        index++;
> +      }
> +      assert (index == -1) || (index > disi.doc());
> +    }
> +    clear(lastNotCleared+1, size());
> +  }
> +
> +  /**
> +   * Perform an inplace NOT with the doc ids from a given  
> DocIdSetIterator,
> +   * clearing all the bits for each such doc id.
> +   * These doc ids should be smaller than the maximum size passed  
> to the
> +   * constructor.
> +   */
> +  public void inPlaceNot(DocIdSetIterator disi) throws IOException {
> +    while (disi.next() && (disi.doc() < size())) {
> +      fastClear(disi.doc());
> +    }
> +  }
> +
> +  /**
> +   * Perform an inplace XOR with the doc ids from a given  
> DocIdSetIterator,
> +   * flipping all the bits for each such doc id.
> +   * These doc ids should be smaller than the maximum size passed  
> to the
> +   * constructor.
> +   */
> +  public void inPlaceXor(DocIdSetIterator disi) throws IOException {
> +    while (disi.next() && (disi.doc() < size())) {
> +      fastFlip(disi.doc());
> +    }
> +  }
> +}
>
> Added: lucene/java/trunk/src/test/org/apache/lucene/search/ 
> OldBitSetFilterWrapper.java
> URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/search/OldBitSetFilterWrapper.java?rev=659635&view=auto
> = 
> = 
> = 
> = 
> = 
> = 
> = 
> = 
> ======================================================================
> --- lucene/java/trunk/src/test/org/apache/lucene/search/ 
> OldBitSetFilterWrapper.java (added)
> +++ lucene/java/trunk/src/test/org/apache/lucene/search/ 
> OldBitSetFilterWrapper.java Fri May 23 12:25:05 2008
> @@ -0,0 +1,48 @@
> +package org.apache.lucene.search;
> +
> +/**
> + * Licensed to the Apache Software Foundation (ASF) under one or more
> + * contributor license agreements.  See the NOTICE file distributed  
> with
> + * this work for additional information regarding copyright  
> ownership.
> + * The ASF licenses this file to You under the Apache License,  
> Version 2.0
> + * (the "License"); you may not use this file except in compliance  
> with
> + * the License.  You may obtain a copy of the License at
> + *
> + *     http://www.apache.org/licenses/LICENSE-2.0
> + *
> + * Unless required by applicable law or agreed to in writing,  
> software
> + * distributed under the License is distributed on an "AS IS" BASIS,
> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or  
> implied.
> + * See the License for the specific language governing permissions  
> and
> + * limitations under the License.
> + */
> +
> +import java.io.IOException;
> +import java.util.BitSet;
> +
> +import org.apache.lucene.index.IndexReader;
> +
> +  /**
> +   *  Helper class used for testing compatibility with old BitSet- 
> based filters.
> +   *  Does not override {@link Filter#getDocIdSet(IndexReader)} and  
> thus ensures
> +   *  that {@link #bits(IndexReader)} is called.
> +   *
> +   *  @deprecated This class will be removed together with the
> +   *  {@link Filter#bits(IndexReader)} method in Lucene 3.0.
> +   */
> +  public class OldBitSetFilterWrapper extends Filter {
> +    private Filter filter;
> +
> +    public OldBitSetFilterWrapper(Filter filter) {
> +      this.filter = filter;
> +    }
> +
> +    public BitSet bits(IndexReader reader) throws IOException {
> +      BitSet bits = new BitSet(reader.maxDoc());
> +      DocIdSetIterator it = filter.getDocIdSet(reader).iterator();
> +      while(it.next()) {
> +        bits.set(it.doc());
> +      }
> +      return bits;
> +    }
> +  }
>
>

--------------------------
Grant Ingersoll
http://www.lucidimagination.com

Lucene Helpful Hints:
http://wiki.apache.org/lucene-java/BasicsOfPerformance
http://wiki.apache.org/lucene-java/LuceneFAQ








---------------------------------------------------------------------
To unsubscribe, e-mail: java-dev-unsubscribe@lucene.apache.org
For additional commands, e-mail: java-dev-help@lucene.apache.org


Mime
View raw message