lucene-java-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From uschind...@apache.org
Subject svn commit: r899359 [5/7] - in /lucene/java/branches/flex_1458: ./ contrib/ contrib/analyzers/common/src/java/org/apache/lucene/analysis/ar/ contrib/analyzers/common/src/java/org/apache/lucene/analysis/bg/ contrib/analyzers/common/src/java/org/apache/l...
Date Thu, 14 Jan 2010 19:05:42 GMT
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/SimpleAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/SimpleAnalyzer.java?rev=899359&r1=899358&r2=899359&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/SimpleAnalyzer.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/SimpleAnalyzer.java Thu Jan 14 19:05:12 2010
@@ -18,25 +18,15 @@
  */
 
 import java.io.Reader;
-import java.io.IOException;
 
 /** An {@link Analyzer} that filters {@link LetterTokenizer} 
  *  with {@link LowerCaseFilter} */
 
-public final class SimpleAnalyzer extends Analyzer {
-  @Override
-  public TokenStream tokenStream(String fieldName, Reader reader) {
-    return new LowerCaseTokenizer(reader);
-  }
+public final class SimpleAnalyzer extends ReusableAnalyzerBase {
 
   @Override
-  public TokenStream reusableTokenStream(String fieldName, Reader reader) throws IOException {
-    Tokenizer tokenizer = (Tokenizer) getPreviousTokenStream();
-    if (tokenizer == null) {
-      tokenizer = new LowerCaseTokenizer(reader);
-      setPreviousTokenStream(tokenizer);
-    } else
-      tokenizer.reset(reader);
-    return tokenizer;
+  protected TokenStreamComponents createComponents(final String fieldName,
+      final Reader reader) {
+    return new TokenStreamComponents(new LowerCaseTokenizer(reader));
   }
 }

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/StopAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/StopAnalyzer.java?rev=899359&r1=899358&r2=899359&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/StopAnalyzer.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/StopAnalyzer.java Thu Jan 14 19:05:12 2010
@@ -24,6 +24,7 @@
 import java.util.Set;
 import java.util.List;
 
+import org.apache.lucene.analysis.ReusableAnalyzerBase.TokenStreamComponents; // javadoc @link
 import org.apache.lucene.util.Version;
 
 /** Filters {@link LetterTokenizer} with {@link LowerCaseFilter} and {@link StopFilter}.
@@ -38,9 +39,7 @@
  * </ul>
 */
 
-public final class StopAnalyzer extends Analyzer {
-  private final Set<?> stopWords;
-  private final Version matchVersion;
+public final class StopAnalyzer extends StopwordAnalyzerBase {
   
   /** An unmodifiable set containing some common English words that are not usually useful
   for searching.*/
@@ -65,16 +64,14 @@
    * @param matchVersion See <a href="#version">above</a>
    */
   public StopAnalyzer(Version matchVersion) {
-    stopWords = ENGLISH_STOP_WORDS_SET;
-    this.matchVersion = matchVersion;
+    this(matchVersion, ENGLISH_STOP_WORDS_SET);
   }
 
   /** Builds an analyzer with the stop words from the given set.
    * @param matchVersion See <a href="#version">above</a>
    * @param stopWords Set of stop words */
   public StopAnalyzer(Version matchVersion, Set<?> stopWords) {
-    this.stopWords = stopWords;
-    this.matchVersion = matchVersion;
+    super(matchVersion, stopWords);
   }
 
   /** Builds an analyzer with the stop words from the given file.
@@ -82,8 +79,7 @@
    * @param matchVersion See <a href="#version">above</a>
    * @param stopwordsFile File to load stop words from */
   public StopAnalyzer(Version matchVersion, File stopwordsFile) throws IOException {
-    stopWords = WordlistLoader.getWordSet(stopwordsFile);
-    this.matchVersion = matchVersion;
+    this(matchVersion, WordlistLoader.getWordSet(stopwordsFile));
   }
 
   /** Builds an analyzer with the stop words from the given reader.
@@ -91,34 +87,21 @@
    * @param matchVersion See <a href="#version">above</a>
    * @param stopwords Reader to load stop words from */
   public StopAnalyzer(Version matchVersion, Reader stopwords) throws IOException {
-    stopWords = WordlistLoader.getWordSet(stopwords);
-    this.matchVersion = matchVersion;
+    this(matchVersion, WordlistLoader.getWordSet(stopwords));
   }
 
-  /** Filters LowerCaseTokenizer with StopFilter. */
-  @Override
-  public TokenStream tokenStream(String fieldName, Reader reader) {
-    return new StopFilter(matchVersion,
-        new LowerCaseTokenizer(reader), stopWords);
-  }
-
-  /** Filters LowerCaseTokenizer with StopFilter. */
-  private class SavedStreams {
-    Tokenizer source;
-    TokenStream result;
-  };
+  /**
+   * Creates {@link TokenStreamComponents} used to tokenize all the text in the provided {@link Reader}.
+   *
+   * @return {@link TokenStreamComponents} built from a {@link LowerCaseTokenizer} filtered with
+   *         {@link StopFilter}
+   */
   @Override
-  public TokenStream reusableTokenStream(String fieldName, Reader reader) throws IOException {
-    SavedStreams streams = (SavedStreams) getPreviousTokenStream();
-    if (streams == null) {
-      streams = new SavedStreams();
-      streams.source = new LowerCaseTokenizer(reader);
-      streams.result = new StopFilter(matchVersion,
-          streams.source, stopWords);
-      setPreviousTokenStream(streams);
-    } else
-      streams.source.reset(reader);
-    return streams.result;
+  protected TokenStreamComponents createComponents(String fieldName,
+      Reader reader) {
+    final Tokenizer source = new LowerCaseTokenizer(reader);
+    return new TokenStreamComponents(source, new StopFilter(matchVersion,
+          source, stopwords));
   }
 }
 

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/StopFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/StopFilter.java?rev=899359&r1=899358&r2=899359&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/StopFilter.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/StopFilter.java Thu Jan 14 19:05:12 2010
@@ -64,6 +64,7 @@
    * @param ignoreCase if true, all words are lower cased first
    * @deprecated use {@link #StopFilter(Version, TokenStream, Set, boolean)} instead
    */
+  @Deprecated
   public StopFilter(boolean enablePositionIncrements, TokenStream input, Set<?> stopWords, boolean ignoreCase)
   {
     this(Version.LUCENE_30, enablePositionIncrements, input, stopWords, ignoreCase);
@@ -101,7 +102,7 @@
    */
   private StopFilter(Version matchVersion, boolean enablePositionIncrements, TokenStream input, Set<?> stopWords, boolean ignoreCase){
     super(input);
-    this.stopWords = CharArraySet.unmodifiableSet(new CharArraySet(matchVersion, stopWords, ignoreCase));
+    this.stopWords = stopWords instanceof CharArraySet ? (CharArraySet)stopWords : new CharArraySet(matchVersion, stopWords, ignoreCase);  
     this.enablePositionIncrements = enablePositionIncrements;
     termAtt = addAttribute(TermAttribute.class);
     posIncrAtt = addAttribute(PositionIncrementAttribute.class);
@@ -117,6 +118,7 @@
    * @see #makeStopSet(Version, java.lang.String[])
    * @deprecated use {@link #StopFilter(Version, TokenStream, Set)} instead
    */
+  @Deprecated
   public StopFilter(boolean enablePositionIncrements, TokenStream in, Set<?> stopWords) {
     this(Version.LUCENE_CURRENT, enablePositionIncrements, in, stopWords, false);
   }
@@ -148,6 +150,7 @@
    * @see #makeStopSet(Version, java.lang.String[], boolean) passing false to ignoreCase
    * @deprecated use {@link #makeStopSet(Version, String...)} instead
    */
+  @Deprecated
   public static final Set<Object> makeStopSet(String... stopWords) {
     return makeStopSet(Version.LUCENE_30, stopWords, false);
   }
@@ -176,6 +179,7 @@
    * @see #makeStopSet(Version, java.lang.String[], boolean) passing false to ignoreCase
    * @deprecated use {@link #makeStopSet(Version, List)} instead
    */
+  @Deprecated
   public static final Set<Object> makeStopSet(List<?> stopWords) {
     return makeStopSet(Version.LUCENE_30, stopWords, false);
   }
@@ -202,6 +206,7 @@
    * @return a Set containing the words
    * @deprecated use {@link #makeStopSet(Version, String[], boolean)} instead;
    */  
+  @Deprecated
   public static final Set<Object> makeStopSet(String[] stopWords, boolean ignoreCase) {
     return makeStopSet(Version.LUCENE_30, stopWords, ignoreCase);
   }
@@ -226,6 +231,7 @@
    * @return A Set ({@link CharArraySet}) containing the words
    * @deprecated use {@link #makeStopSet(Version, List, boolean)} instead
    */
+  @Deprecated
   public static final Set<Object> makeStopSet(List<?> stopWords, boolean ignoreCase){
     return makeStopSet(Version.LUCENE_30, stopWords, ignoreCase);
   }
@@ -271,6 +277,7 @@
    * or later, it returns true.
    * @deprecated use {@link #StopFilter(Version, TokenStream, Set)} instead
    */
+  @Deprecated
   public static boolean getEnablePositionIncrementsVersionDefault(Version matchVersion) {
     return matchVersion.onOrAfter(Version.LUCENE_29);
   }

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/Tokenizer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/Tokenizer.java?rev=899359&r1=899358&r2=899359&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/Tokenizer.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/Tokenizer.java Thu Jan 14 19:05:12 2010
@@ -29,8 +29,6 @@
   NOTE: Subclasses overriding {@link #incrementToken()} must
   call {@link AttributeSource#clearAttributes()} before
   setting attributes.
-  Subclasses overriding {@link #incrementToken()} must call
-  {@link Token#clear()} before setting Token attributes. 
  */
 public abstract class Tokenizer extends TokenStream {
   /** The text source for this Tokenizer. */

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/WhitespaceAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/WhitespaceAnalyzer.java?rev=899359&r1=899358&r2=899359&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/WhitespaceAnalyzer.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/WhitespaceAnalyzer.java Thu Jan 14 19:05:12 2010
@@ -18,24 +18,14 @@
  */
 
 import java.io.Reader;
-import java.io.IOException;
 
 /** An Analyzer that uses {@link WhitespaceTokenizer}. */
 
-public final class WhitespaceAnalyzer extends Analyzer {
-  @Override
-  public TokenStream tokenStream(String fieldName, Reader reader) {
-    return new WhitespaceTokenizer(reader);
-  }
+public final class WhitespaceAnalyzer extends ReusableAnalyzerBase {
 
   @Override
-  public TokenStream reusableTokenStream(String fieldName, Reader reader) throws IOException {
-    Tokenizer tokenizer = (Tokenizer) getPreviousTokenStream();
-    if (tokenizer == null) {
-      tokenizer = new WhitespaceTokenizer(reader);
-      setPreviousTokenStream(tokenizer);
-    } else
-      tokenizer.reset(reader);
-    return tokenizer;
+  protected TokenStreamComponents createComponents(final String fieldName,
+      final Reader reader) {
+    return new TokenStreamComponents(new WhitespaceTokenizer(reader));
   }
 }

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/WordlistLoader.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/WordlistLoader.java?rev=899359&r1=899358&r2=899359&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/WordlistLoader.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/WordlistLoader.java Thu Jan 14 19:05:12 2010
@@ -21,15 +21,69 @@
 import java.io.File;
 import java.io.FileReader;
 import java.io.IOException;
+import java.io.InputStreamReader;
 import java.io.Reader;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.Set;
 
 /**
  * Loader for text files that represent a list of stopwords.
  */
 public class WordlistLoader {
-
+ 
+  /**
+   * Loads a text file associated with a given class (See
+   * {@link Class#getResourceAsStream(String)}) and adds every line as an entry
+   * to a {@link Set} (omitting leading and trailing whitespace). Every line of
+   * the file should contain only one word. The words need to be in lower-case if
+   * you make use of an Analyzer which uses LowerCaseFilter (like
+   * StandardAnalyzer).
+   * 
+   * @param aClass
+   *          a class that is associated with the given stopwordResource
+   * @param stopwordResource
+   *          name of the resource file associated with the given class
+   * @return a {@link Set} with the file's words
+   */
+  public static Set<String> getWordSet(Class<?> aClass, String stopwordResource)
+      throws IOException {
+    final Reader reader = new BufferedReader(new InputStreamReader(aClass
+        .getResourceAsStream(stopwordResource), "UTF-8"));
+    try {
+      return getWordSet(reader);
+    } finally {
+      reader.close();
+    }
+  }
+  
+  /**
+   * Loads a text file associated with a given class (See
+   * {@link Class#getResourceAsStream(String)}) and adds every line as an entry
+   * to a {@link Set} (omitting leading and trailing whitespace). Every line of
+   * the file should contain only one word. The words need to be in lower-case if
+   * you make use of an Analyzer which uses LowerCaseFilter (like
+   * StandardAnalyzer).
+   * 
+   * @param aClass
+   *          a class that is associated with the given stopwordResource
+   * @param stopwordResource
+   *          name of the resource file associated with the given class
+   * @param comment
+   *          the comment string to ignore
+   * @return a {@link Set} with the file's words
+   */
+  public static Set<String> getWordSet(Class<?> aClass,
+      String stopwordResource, String comment) throws IOException {
+    final Reader reader = new BufferedReader(new InputStreamReader(aClass
+        .getResourceAsStream(stopwordResource), "UTF-8"));
+    try {
+      return getWordSet(reader, comment);
+    } finally {
+      reader.close();
+    }
+  }
+  
   /**
    * Loads a text file and adds every line as an entry to a HashSet (omitting
    * leading and trailing whitespace). Every line of the file should contain only
@@ -40,17 +94,15 @@
    * @return A HashSet with the file's words
    */
   public static HashSet<String> getWordSet(File wordfile) throws IOException {
-    HashSet<String> result = new HashSet<String>();
     FileReader reader = null;
     try {
       reader = new FileReader(wordfile);
-      result = getWordSet(reader);
+      return getWordSet(reader);
     }
     finally {
       if (reader != null)
         reader.close();
     }
-    return result;
   }
 
   /**
@@ -64,17 +116,15 @@
    * @return A HashSet with the file's words
    */
   public static HashSet<String> getWordSet(File wordfile, String comment) throws IOException {
-    HashSet<String> result = new HashSet<String>();
     FileReader reader = null;
     try {
       reader = new FileReader(wordfile);
-      result = getWordSet(reader, comment);
+      return getWordSet(reader, comment);
     }
     finally {
       if (reader != null)
         reader.close();
     }
-    return result;
   }
 
 
@@ -88,7 +138,7 @@
    * @return A HashSet with the reader's words
    */
   public static HashSet<String> getWordSet(Reader reader) throws IOException {
-    HashSet<String> result = new HashSet<String>();
+    final HashSet<String> result = new HashSet<String>();
     BufferedReader br = null;
     try {
       if (reader instanceof BufferedReader) {
@@ -119,7 +169,7 @@
    * @return A HashSet with the reader's words
    */
   public static HashSet<String> getWordSet(Reader reader, String comment) throws IOException {
-    HashSet<String> result = new HashSet<String>();
+    final HashSet<String> result = new HashSet<String>();
     BufferedReader br = null;
     try {
       if (reader instanceof BufferedReader) {
@@ -146,7 +196,7 @@
   /**
    * Reads a stem dictionary. Each line contains:
    * <pre>word<b>\t</b>stem</pre>
-   * (i.e. two tab seperated words)
+   * (i.e. two tab separated words)
    *
    * @return stem dictionary that overrules the stemming algorithm
    * @throws IOException 
@@ -154,21 +204,18 @@
   public static HashMap<String, String> getStemDict(File wordstemfile) throws IOException {
     if (wordstemfile == null)
       throw new NullPointerException("wordstemfile may not be null");
-    HashMap<String, String> result = new HashMap<String, String>();
+    final HashMap<String, String> result = new HashMap<String,String>();
     BufferedReader br = null;
-    FileReader fr = null;
+    
     try {
-      fr = new FileReader(wordstemfile);
-      br = new BufferedReader(fr);
+      br = new BufferedReader(new FileReader(wordstemfile));
       String line;
       while ((line = br.readLine()) != null) {
         String[] wordstem = line.split("\t", 2);
         result.put(wordstem[0], wordstem[1]);
       }
     } finally {
-      if (fr != null)
-        fr.close();
-      if (br != null)
+      if(br != null)
         br.close();
     }
     return result;

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/standard/StandardAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/standard/StandardAnalyzer.java?rev=899359&r1=899358&r2=899359&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/standard/StandardAnalyzer.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/standard/StandardAnalyzer.java Thu Jan 14 19:05:12 2010
@@ -71,7 +71,6 @@
    * @param stopWords stop words */
   public StandardAnalyzer(Version matchVersion, Set<?> stopWords) {
     stopSet = stopWords;
-    setOverridesTokenStreamMethod(StandardAnalyzer.class);
     replaceInvalidAcronym = matchVersion.onOrAfter(Version.LUCENE_24);
     this.matchVersion = matchVersion;
   }

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java?rev=899359&r1=899358&r2=899359&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java Thu Jan 14 19:05:12 2010
@@ -71,6 +71,7 @@
    * @deprecated this solves a bug where HOSTs that end with '.' are identified
    *             as ACRONYMs.
    */
+  @Deprecated
   public static final int ACRONYM_DEP       = 8;
 
   /** String token types that correspond to token type int constants */
@@ -227,6 +228,7 @@
    *
    * @deprecated Remove in 3.X and make true the only valid value
    */
+  @Deprecated
   public boolean isReplaceInvalidAcronym() {
     return replaceInvalidAcronym;
   }
@@ -238,6 +240,7 @@
    *
    * See https://issues.apache.org/jira/browse/LUCENE-1068
    */
+  @Deprecated
   public void setReplaceInvalidAcronym(boolean replaceInvalidAcronym) {
     this.replaceInvalidAcronym = replaceInvalidAcronym;
   }

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.java?rev=899359&r1=899358&r2=899359&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.java Thu Jan 14 19:05:12 2010
@@ -354,6 +354,7 @@
  * @deprecated this solves a bug where HOSTs that end with '.' are identified
  *             as ACRONYMs.
  */
+@Deprecated
 public static final int ACRONYM_DEP       = StandardTokenizer.ACRONYM_DEP;
 
 public static final String [] TOKEN_TYPES = StandardTokenizer.TOKEN_TYPES;

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.jflex
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.jflex?rev=899359&r1=899358&r2=899359&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.jflex (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.jflex Thu Jan 14 19:05:12 2010
@@ -55,6 +55,7 @@
  * @deprecated this solves a bug where HOSTs that end with '.' are identified
  *             as ACRONYMs.
  */
+@Deprecated
 public static final int ACRONYM_DEP       = StandardTokenizer.ACRONYM_DEP;
 
 public static final String [] TOKEN_TYPES = StandardTokenizer.TOKEN_TYPES;

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/collation/CollationKeyFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/collation/CollationKeyFilter.java?rev=899359&r1=899358&r2=899359&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/collation/CollationKeyFilter.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/collation/CollationKeyFilter.java Thu Jan 14 19:05:12 2010
@@ -24,8 +24,6 @@
 import org.apache.lucene.util.IndexableBinaryStringTools;
 
 import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.nio.CharBuffer;
 import java.text.Collator;
 
 
@@ -94,15 +92,14 @@
       char[] termBuffer = termAtt.termBuffer();
       String termText = new String(termBuffer, 0, termAtt.termLength());
       byte[] collationKey = collator.getCollationKey(termText).toByteArray();
-      ByteBuffer collationKeyBuf = ByteBuffer.wrap(collationKey);
-      int encodedLength
-        = IndexableBinaryStringTools.getEncodedLength(collationKeyBuf);
+      int encodedLength = IndexableBinaryStringTools.getEncodedLength(
+          collationKey, 0, collationKey.length);
       if (encodedLength > termBuffer.length) {
         termAtt.resizeTermBuffer(encodedLength);
       }
       termAtt.setTermLength(encodedLength);
-      CharBuffer wrappedTermBuffer = CharBuffer.wrap(termAtt.termBuffer());
-      IndexableBinaryStringTools.encode(collationKeyBuf, wrappedTermBuffer);
+      IndexableBinaryStringTools.encode(collationKey, 0, collationKey.length,
+          termAtt.termBuffer(), 0, encodedLength);
       return true;
     } else {
       return false;

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/document/DateField.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/document/DateField.java?rev=899359&r1=899358&r2=899359&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/document/DateField.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/document/DateField.java Thu Jan 14 19:05:12 2010
@@ -57,6 +57,7 @@
  * This class is included for use with existing
  * indices and will be removed in a future release (possibly Lucene 4.0).
  */
+@Deprecated
 public class DateField {
   
   private DateField() {}

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/document/Field.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/document/Field.java?rev=899359&r1=899358&r2=899359&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/document/Field.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/document/Field.java Thu Jan 14 19:05:12 2010
@@ -521,11 +521,27 @@
    * 
    * @param name The name of the field
    * @param value The binary value
-   * @param store How <code>value</code> should be stored (compressed or not)
+   * @param store Must be Store.YES
    * @throws IllegalArgumentException if store is <code>Store.NO</code> 
+   * @deprecated Use {@link #Field(String, byte[]) instead}
    */
+  @Deprecated
   public Field(String name, byte[] value, Store store) {
-    this(name, value, 0, value.length, store);
+    this(name, value, 0, value.length);
+
+    if (store == Store.NO) {
+      throw new IllegalArgumentException("binary values can't be unstored");
+    }
+  }
+
+  /**
+   * Create a stored field with binary value. Optionally the value may be compressed.
+   * 
+   * @param name The name of the field
+   * @param value The binary value
+   */
+  public Field(String name, byte[] value) {
+    this(name, value, 0, value.length);
   }
 
   /**
@@ -537,8 +553,26 @@
    * @param length Number of bytes to use for this Field, starting at offset
    * @param store How <code>value</code> should be stored (compressed or not)
    * @throws IllegalArgumentException if store is <code>Store.NO</code> 
+   * @deprecated Use {@link #Field(String, byte[], int, int) instead}
    */
+  @Deprecated
   public Field(String name, byte[] value, int offset, int length, Store store) {
+    this(name, value, offset, length);
+
+    if (store == Store.NO) {
+      throw new IllegalArgumentException("binary values can't be unstored");
+    }
+  }
+
+  /**
+   * Create a stored field with binary value. Optionally the value may be compressed.
+   * 
+   * @param name The name of the field
+   * @param value The binary value
+   * @param offset Starting offset in value where this Field's bytes are
+   * @param length Number of bytes to use for this Field, starting at offset
+   */
+  public Field(String name, byte[] value, int offset, int length) {
 
     if (name == null)
       throw new IllegalArgumentException("name cannot be null");
@@ -548,10 +582,7 @@
     this.name = StringHelper.intern(name);        // field names are interned
     fieldsData = value;
     
-    if (store == Store.NO)
-      throw new IllegalArgumentException("binary values can't be unstored");
-    
-    isStored = store.isStored();
+    isStored = true;
     isIndexed   = false;
     isTokenized = false;
     omitTermFreqAndPositions = false;

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/document/NumberTools.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/document/NumberTools.java?rev=899359&r1=899358&r2=899359&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/document/NumberTools.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/document/NumberTools.java Thu Jan 14 19:05:12 2010
@@ -44,6 +44,7 @@
  * This class is included for use with existing
  * indices and will be removed in a future release (possibly Lucene 4.0).
  */
+@Deprecated
 public class NumberTools {
 
     private static final int RADIX = 36;

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/AllTermDocs.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/AllTermDocs.java?rev=899359&r1=899358&r2=899359&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/AllTermDocs.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/AllTermDocs.java Thu Jan 14 19:05:12 2010
@@ -31,6 +31,7 @@
     }
   }
 
+  @Override
   public boolean isDeleted(int doc) {
     return deletedDocs != null && deletedDocs.get(doc);
   }

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java?rev=899359&r1=899358&r2=899359&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java Thu Jan 14 19:05:12 2010
@@ -23,24 +23,45 @@
 import java.io.IOException;
 import java.util.List;
 import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.Collections;
 
 /** A {@link MergeScheduler} that runs each merge using a
- *  separate thread, up until a maximum number of threads
- *  ({@link #setMaxThreadCount}) at which when a merge is
- *  needed, the thread(s) that are updating the index will
- *  pause until one or more merges completes.  This is a
- *  simple way to use concurrency in the indexing process
- *  without having to create and manage application level
- *  threads. */
-
+ *  separate thread.
+ *
+ *  <p>Specify the max number of threads that may run at
+ *  once with {@link #setMaxThreadCount}.</p>
+ *
+ *  <p>Separately specify the maximum number of simultaneous
+ *  merges with {@link #setMaxMergeCount}.  If the number of
+ *  merges exceeds the max number of threads then the
+ *  largest merges are paused until one of the smaller
+ *  merges completes.</p>
+ *
+ *  <p>If more than {@link #getMaxMergeCount} merges are
+ *  requested then this class will forcefully throttle the
+ *  incoming threads by pausing until one more more merges
+ *  complete.</p>
+ */ 
 public class ConcurrentMergeScheduler extends MergeScheduler {
 
   private int mergeThreadPriority = -1;
 
   protected List<MergeThread> mergeThreads = new ArrayList<MergeThread>();
 
-  // Max number of threads allowed to be merging at once
-  private int maxThreadCount = 1;
+  // Max number of merge threads allowed to be running at
+  // once.  When there are more merges then this, we
+  // forcefully pause the larger ones, letting the smaller
+  // ones run, up until maxMergeCount merges at which point
+  // we forcefully pause incoming threads (that presumably
+  // are the ones causing so much merging).  We dynamically
+  // default this from 1 to 3, depending on how many cores
+  // you have:
+  private int maxThreadCount = Math.max(1, Math.min(3, Runtime.getRuntime().availableProcessors()/2));
+
+  // Max number of merges we accept before forcefully
+  // throttling the incoming threads
+  private int maxMergeCount = maxThreadCount+2;
 
   protected Directory dir;
 
@@ -55,23 +76,45 @@
     }
   }
 
-  /** Sets the max # simultaneous threads that may be
-   *  running.  If a merge is necessary yet we already have
-   *  this many threads running, the incoming thread (that
-   *  is calling add/updateDocument) will block until
-   *  a merge thread has completed. */
+  /** Sets the max # simultaneous merge threads that should
+   *  be running at once.  This must be <= {@link
+   *  #setMaxMergeCount}. */
   public void setMaxThreadCount(int count) {
-    if (count < 1)
+    if (count < 1) {
       throw new IllegalArgumentException("count should be at least 1");
+    }
+    if (count > maxMergeCount) {
+      throw new IllegalArgumentException("count should be <= maxMergeCount (= " + maxMergeCount + ")");
+    }
     maxThreadCount = count;
   }
 
-  /** Get the max # simultaneous threads that may be
-   *  running. @see #setMaxThreadCount. */
+  /** @see #setMaxThreadCount(int) */
   public int getMaxThreadCount() {
     return maxThreadCount;
   }
 
+  /** Sets the max # simultaneous merges that are allowed.
+   *  If a merge is necessary yet we already have this many
+   *  threads running, the incoming thread (that is calling
+   *  add/updateDocument) will block until a merge thread
+   *  has completed.  Note that we will only run the
+   *  smallest {@link #setMaxThreadCount} merges at a time. */
+  public void setMaxMergeCount(int count) {
+    if (count < 1) {
+      throw new IllegalArgumentException("count should be at least 1");
+    }
+    if (count < maxThreadCount) {
+      throw new IllegalArgumentException("count should be >= maxThreadCount (= " + maxThreadCount + ")");
+    }
+    maxMergeCount = count;
+  }
+
+  /** See {@link #setMaxMergeCount}. */
+  public int getMaxMergeCount() {
+    return maxMergeCount;
+  }
+
   /** Return the priority that merge threads run at.  By
    *  default the priority is 1 plus the priority of (ie,
    *  slightly higher priority than) the first thread that
@@ -81,16 +124,73 @@
     return mergeThreadPriority;
   }
 
-  /** Set the priority that merge threads run at. */
+  /** Set the base priority that merge threads run at.
+   *  Note that CMS may increase priority of some merge
+   *  threads beyond this base priority.  It's best not to
+   *  set this any higher than
+   *  Thread.MAX_PRIORITY-maxThreadCount, so that CMS has
+   *  room to set relative priority among threads.  */
   public synchronized void setMergeThreadPriority(int pri) {
     if (pri > Thread.MAX_PRIORITY || pri < Thread.MIN_PRIORITY)
       throw new IllegalArgumentException("priority must be in range " + Thread.MIN_PRIORITY + " .. " + Thread.MAX_PRIORITY + " inclusive");
     mergeThreadPriority = pri;
+    updateMergeThreads();
+  }
 
-    final int numThreads = mergeThreadCount();
-    for(int i=0;i<numThreads;i++) {
-      MergeThread merge = mergeThreads.get(i);
-      merge.setThreadPriority(pri);
+  // Larger merges come first
+  protected static class CompareByMergeDocCount implements Comparator<MergeThread> {
+    public int compare(MergeThread t1, MergeThread t2) {
+      final MergePolicy.OneMerge m1 = t1.getCurrentMerge();
+      final MergePolicy.OneMerge m2 = t2.getCurrentMerge();
+      
+      final int c1 = m1 == null ? Integer.MAX_VALUE : m1.segments.totalDocCount();
+      final int c2 = m2 == null ? Integer.MAX_VALUE : m2.segments.totalDocCount();
+
+      return c2 - c1;
+    }
+  }
+
+  /** Called whenever the running merges have changed, to
+   *  pause & unpause threads. */
+  protected synchronized void updateMergeThreads() {
+
+    Collections.sort(mergeThreads, new CompareByMergeDocCount());
+    
+    final int count = mergeThreads.size();
+    int pri = mergeThreadPriority;
+    for(int i=0;i<count;i++) {
+      final MergeThread mergeThread = mergeThreads.get(i);
+      final MergePolicy.OneMerge merge = mergeThread.getCurrentMerge();
+      if (merge == null) {
+        continue;
+      }
+      final boolean doPause;
+      if (i < count-maxThreadCount) {
+        doPause = true;
+      } else {
+        doPause = false;
+      }
+
+      if (verbose()) {
+        if (doPause != merge.getPause()) {
+          if (doPause) {
+            message("pause thread " + mergeThread.getName());
+          } else {
+            message("unpause thread " + mergeThread.getName());
+          }
+        }
+      }
+      if (doPause != merge.getPause()) {
+        merge.setPause(doPause);
+      }
+
+      if (!doPause) {
+        if (verbose()) {
+          message("set priority of merge thread " + mergeThread.getName() + " to " + pri);
+        }
+        mergeThread.setThreadPriority(pri);
+        pri = Math.min(Thread.MAX_PRIORITY, 1+pri);
+      }
     }
   }
 
@@ -192,9 +292,12 @@
       try {
         synchronized(this) {
           final MergeThread merger;
-          while (mergeThreadCount() >= maxThreadCount) {
-            if (verbose())
-              message("    too many merge threads running; stalling...");
+          long startStallTime = 0;
+          while (mergeThreadCount() >= maxMergeCount) {
+            startStallTime = System.currentTimeMillis();
+            if (verbose()) {
+              message("    too many merges; stalling...");
+            }
             try {
               wait();
             } catch (InterruptedException ie) {
@@ -202,15 +305,20 @@
             }
           }
 
-          if (verbose())
+          if (verbose()) {
+            if (startStallTime != 0) {
+              message("  stalled for " + (System.currentTimeMillis()-startStallTime) + " msec");
+            }
             message("  consider merge " + merge.segString(dir));
-      
-          assert mergeThreadCount() < maxThreadCount;
+          }
+
+          assert mergeThreadCount() < maxMergeCount;
 
           // OK to spawn a new merge thread to handle this
           // merge:
           merger = getMergeThread(writer, merge);
           mergeThreads.add(merger);
+          updateMergeThreads();
           if (verbose())
             message("    launch new thread [" + merger.getName() + "]");
 
@@ -245,6 +353,7 @@
     IndexWriter writer;
     MergePolicy.OneMerge startMerge;
     MergePolicy.OneMerge runningMerge;
+    private volatile boolean done;
 
     public MergeThread(IndexWriter writer, MergePolicy.OneMerge startMerge) throws IOException {
       this.writer = writer;
@@ -259,6 +368,16 @@
       return runningMerge;
     }
 
+    public synchronized MergePolicy.OneMerge getCurrentMerge() {
+      if (done) {
+        return null;
+      } else if (runningMerge != null) {
+        return runningMerge;
+      } else {
+        return startMerge;
+      }
+    }
+
     public void setThreadPriority(int pri) {
       try {
         setPriority(pri);
@@ -292,10 +411,14 @@
           merge = writer.getNextMerge();
           if (merge != null) {
             writer.mergeInit(merge);
+            updateMergeThreads();
             if (verbose())
               message("  merge thread: do another merge " + merge.segString(dir));
-          } else
+          } else {
+            done = true;
+            updateMergeThreads();
             break;
+          }
         }
 
         if (verbose())
@@ -317,6 +440,7 @@
           ConcurrentMergeScheduler.this.notifyAll();
           boolean removed = mergeThreads.remove(this);
           assert removed;
+          updateMergeThreads();
         }
       }
     }

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DirectoryReader.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DirectoryReader.java?rev=899359&r1=899358&r2=899359&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DirectoryReader.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DirectoryReader.java Thu Jan 14 19:05:12 2010
@@ -32,7 +32,6 @@
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.FieldSelector;
 import org.apache.lucene.search.Similarity;
-import org.apache.lucene.store.AlreadyClosedException;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.Lock;
 import org.apache.lucene.store.LockObtainFailedException;
@@ -41,6 +40,8 @@
 import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.ReaderUtil;
 
+import org.apache.lucene.search.FieldCache; // not great (circular); used only to purge FieldCache entry on close
+
 /** 
  * An IndexReader which reads indexes with multiple segments.
  */
@@ -154,7 +155,7 @@
   DirectoryReader(IndexWriter writer, SegmentInfos infos, int termInfosIndexDivisor, Codecs codecs) throws IOException {
     this.directory = writer.getDirectory();
     this.readOnly = true;
-    this.segmentInfos = infos;
+    segmentInfos = infos;
     this.termInfosIndexDivisor = termInfosIndexDivisor;
     if (codecs == null) {
       this.codecs = Codecs.getDefault();
@@ -340,6 +341,22 @@
     }
   }
 
+  /** {@inheritDoc} */
+  @Override
+  public String toString() {
+    final StringBuilder buffer = new StringBuilder();
+    if (hasChanges) {
+      buffer.append("*");
+    }
+    buffer.append(getClass().getSimpleName());
+    buffer.append('(');
+    for(SegmentReader r : subReaders) {
+      buffer.append(r);
+    }
+    buffer.append(')');
+    return buffer.toString();
+  }
+
   private void initialize(SegmentReader[] subReaders) {
     this.subReaders = subReaders;
     starts = new int[subReaders.length + 1];    // build starts array
@@ -456,22 +473,39 @@
   }
 
   @Override
-  public final synchronized IndexReader reopen() throws CorruptIndexException, IOException {
+  public final IndexReader reopen() throws CorruptIndexException, IOException {
     // Preserve current readOnly
     return doReopen(readOnly, null);
   }
 
   @Override
-  public final synchronized IndexReader reopen(boolean openReadOnly) throws CorruptIndexException, IOException {
+  public final IndexReader reopen(boolean openReadOnly) throws CorruptIndexException, IOException {
     return doReopen(openReadOnly, null);
   }
 
   @Override
-  public final synchronized IndexReader reopen(final IndexCommit commit) throws CorruptIndexException, IOException {
+  public final IndexReader reopen(final IndexCommit commit) throws CorruptIndexException, IOException {
     return doReopen(true, commit);
   }
 
-  private synchronized IndexReader doReopen(final boolean openReadOnly, IndexCommit commit) throws CorruptIndexException, IOException {
+  private final IndexReader doReopenFromWriter(boolean openReadOnly, IndexCommit commit) throws CorruptIndexException, IOException {
+    assert readOnly;
+
+    if (!openReadOnly) {
+      throw new IllegalArgumentException("a reader obtained from IndexWriter.getReader() can only be reopened with openReadOnly=true (got false)");
+    }
+
+    if (commit != null) {
+      throw new IllegalArgumentException("a reader obtained from IndexWriter.getReader() cannot currently accept a commit");
+    }
+
+    // TODO: right now we *always* make a new reader; in
+    // the future we could have write make some effort to
+    // detect that no changes have occurred
+    return writer.getReader();
+  }
+
+  private IndexReader doReopen(final boolean openReadOnly, IndexCommit commit) throws CorruptIndexException, IOException {
     ensureOpen();
 
     assert commit == null || openReadOnly;
@@ -479,26 +513,13 @@
     // If we were obtained by writer.getReader(), re-ask the
     // writer to get a new reader.
     if (writer != null) {
-      assert readOnly;
-
-      if (!openReadOnly) {
-        throw new IllegalArgumentException("a reader obtained from IndexWriter.getReader() can only be reopened with openReadOnly=true (got false)");
-      }
-
-      if (commit != null) {
-        throw new IllegalArgumentException("a reader obtained from IndexWriter.getReader() cannot currently accept a commit");
-      }
-
-      if (!writer.isOpen(true)) {
-        throw new AlreadyClosedException("cannot reopen: the IndexWriter this reader was obtained from is now closed");
-      }
-
-      // TODO: right now we *always* make a new reader; in
-      // the future we could have write make some effort to
-      // detect that no changes have occurred
-      IndexReader reader = writer.getReader();
-      return reader;
+      return doReopenFromWriter(openReadOnly, commit);
+    } else {
+      return doReopenNoWriter(openReadOnly, commit);
     }
+  }
+
+  private synchronized IndexReader doReopenNoWriter(final boolean openReadOnly, IndexCommit commit) throws CorruptIndexException, IOException {
 
     if (commit == null) {
       if (hasChanges) {
@@ -602,10 +623,13 @@
     ensureOpen();
     return segmentInfos.size() == 1 && !hasDeletions();
   }
-  
+
   @Override
-  public synchronized int numDocs() {
+  public int numDocs() {
     // Don't call ensureOpen() here (it could affect performance)
+
+    // NOTE: multiple threads may wind up init'ing
+    // numDocs... but that's harmless
     if (numDocs == -1) {        // check cache
       int n = 0;                // cache miss--recompute
       for (int i = 0; i < subReaders.length; i++)
@@ -991,6 +1015,12 @@
         if (ioe == null) ioe = e;
       }
     }
+
+    // NOTE: only needed in case someone had asked for
+    // FieldCache for top-level reader (which is generally
+    // not a good idea):
+    FieldCache.DEFAULT.purge(this);
+
     // throw the first exception
     if (ioe != null) throw ioe;
   }
@@ -1025,6 +1055,7 @@
     return directory;
   }
 
+  @Override
   public int getTermInfosIndexDivisor() {
     return termInfosIndexDivisor;
   }

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DocFieldProcessorPerThread.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DocFieldProcessorPerThread.java?rev=899359&r1=899358&r2=899359&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DocFieldProcessorPerThread.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DocFieldProcessorPerThread.java Thu Jan 14 19:05:12 2010
@@ -245,8 +245,10 @@
     for(int i=0;i<fieldCount;i++)
       fields[i].consumer.processFields(fields[i].fields, fields[i].fieldCount);
 
-    if (docState.maxTermPrefix != null && docState.infoStream != null)
+    if (docState.maxTermPrefix != null && docState.infoStream != null) {
       docState.infoStream.println("WARNING: document contains at least one immense term (whose UTF8 encoding is longer than the max length " + DocumentsWriter.MAX_TERM_LENGTH_UTF8 + "), all of which were skipped.  Please correct the analyzer to not produce such terms.  The prefix of the first immense term is: '" + docState.maxTermPrefix + "...'"); 
+      docState.maxTermPrefix = null;
+    }
 
     final DocumentsWriter.DocWriter one = fieldsWriter.finishDocument();
     final DocumentsWriter.DocWriter two = consumer.finishDocument();
@@ -386,4 +388,4 @@
       }
     }
   }
-}
+}
\ No newline at end of file

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/FieldsReader.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/FieldsReader.java?rev=899359&r1=899358&r2=899359&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/FieldsReader.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/FieldsReader.java Thu Jan 14 19:05:12 2010
@@ -164,7 +164,7 @@
   /**
    * @throws AlreadyClosedException if this FieldsReader is closed
    */
-  protected final void ensureOpen() throws AlreadyClosedException {
+  private void ensureOpen() throws AlreadyClosedException {
     if (closed) {
       throw new AlreadyClosedException("this FieldsReader is closed");
     }
@@ -412,6 +412,7 @@
     private int toRead;
     private long pointer;
     /** @deprecated Only kept for backward-compatbility with <3.0 indexes. Will be removed in 4.0. */
+    @Deprecated
     private boolean isCompressed;
 
     public LazyField(String name, Field.Store store, int toRead, long pointer, boolean isBinary, boolean isCompressed) {

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/FieldsWriter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/FieldsWriter.java?rev=899359&r1=899358&r2=899359&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/FieldsWriter.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/FieldsWriter.java Thu Jan 14 19:05:12 2010
@@ -32,6 +32,7 @@
   static final byte FIELD_IS_BINARY = 0x2;
   
   /** @deprecated Kept for backwards-compatibility with <3.0 indexes; will be removed in 4.0 */
+  @Deprecated
   static final byte FIELD_IS_COMPRESSED = 0x4;
 
   // Original format

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/FilterIndexReader.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/FilterIndexReader.java?rev=899359&r1=899358&r2=899359&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/FilterIndexReader.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/FilterIndexReader.java Thu Jan 14 19:05:12 2010
@@ -21,6 +21,7 @@
 import org.apache.lucene.document.FieldSelector;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.util.Bits;
+import org.apache.lucene.search.FieldCache; // not great (circular); used only to purge FieldCache entry on close
 
 import java.io.IOException;
 import java.util.Collection;
@@ -253,7 +254,14 @@
   protected void doCommit(Map<String,String> commitUserData) throws IOException { in.commit(commitUserData); }
   
   @Override
-  protected void doClose() throws IOException { in.close(); }
+  protected void doClose() throws IOException {
+    in.close();
+
+    // NOTE: only needed in case someone had asked for
+    // FieldCache for top-level reader (which is generally
+    // not a good idea):
+    FieldCache.DEFAULT.purge(this);
+  }
 
 
   @Override
@@ -284,4 +292,21 @@
   public IndexReader[] getSequentialSubReaders() {
     return in.getSequentialSubReaders();
   }
-}
+
+  /** If the subclass of FilteredIndexReader modifies the
+   *  contents of the FieldCache, you must override this
+   *  method to provide a different key */
+  @Override
+  public Object getFieldCacheKey() {
+    return in.getFieldCacheKey();
+  }
+
+  /** {@inheritDoc} */
+  @Override
+  public String toString() {
+    final StringBuilder buffer = new StringBuilder("FilterReader(");
+    buffer.append(in);
+    buffer.append(')');
+    return buffer.toString();
+  }
+}
\ No newline at end of file

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/IndexFileDeleter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/IndexFileDeleter.java?rev=899359&r1=899358&r2=899359&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/IndexFileDeleter.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/IndexFileDeleter.java Thu Jan 14 19:05:12 2010
@@ -537,7 +537,7 @@
         // the file for subsequent deletion.
 
         if (infoStream != null) {
-          message("IndexFileDeleter: unable to remove file \"" + fileName + "\": " + e.toString() + "; Will re-try later.");
+          message("unable to remove file \"" + fileName + "\": " + e.toString() + "; Will re-try later.");
         }
         if (deletable == null) {
           deletable = new ArrayList<String>();

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/IndexReader.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/IndexReader.java?rev=899359&r1=899358&r2=899359&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/IndexReader.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/IndexReader.java Thu Jan 14 19:05:12 2010
@@ -32,6 +32,7 @@
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
 
 /** IndexReader is an abstract class, providing an interface for accessing an
  index.  Search of an index is done entirely through this abstract interface,
@@ -119,13 +120,13 @@
   private boolean closed;
   protected boolean hasChanges;
   
-  private int refCount;
+  private final AtomicInteger refCount = new AtomicInteger();
 
   static int DEFAULT_TERMS_INDEX_DIVISOR = 1;
 
   /** Expert: returns the current refCount for this reader */
-  public synchronized int getRefCount() {
-    return refCount;
+  public int getRefCount() {
+    return refCount.get();
   }
   
   /**
@@ -142,41 +143,68 @@
    *
    * @see #decRef
    */
-  public synchronized void incRef() {
-    assert refCount > 0;
+  public void incRef() {
     ensureOpen();
-    refCount++;
+    refCount.incrementAndGet();
+  }
+
+  /** {@inheritDoc} */
+  @Override
+  public String toString() {
+    final StringBuilder buffer = new StringBuilder();
+    if (hasChanges) {
+      buffer.append('*');
+    }
+    buffer.append(getClass().getSimpleName());
+    buffer.append('(');
+    final IndexReader[] subReaders = getSequentialSubReaders();
+    if ((subReaders != null) && (subReaders.length > 0)) {
+      buffer.append(subReaders[0]);
+      for (int i = 1; i < subReaders.length; ++i) {
+        buffer.append(" ").append(subReaders[i]);
+      }
+    }
+    buffer.append(')');
+    return buffer.toString();
   }
 
   /**
    * Expert: decreases the refCount of this IndexReader
    * instance.  If the refCount drops to 0, then pending
    * changes (if any) are committed to the index and this
-   * reader is closed.
-   * 
+   * reader is closed.  If an exception is hit, the refCount
+   * is unchanged.
+   *
    * @throws IOException in case an IOException occurs in commit() or doClose()
    *
    * @see #incRef
    */
-  public synchronized void decRef() throws IOException {
-    assert refCount > 0;
+  public void decRef() throws IOException {
     ensureOpen();
-    if (refCount == 1) {
-      commit();
-      doClose();
+    if (refCount.getAndDecrement() == 1) {
+      boolean success = false;
+      try {
+        commit();
+        doClose();
+        success = true;
+      } finally {
+        if (!success) {
+          // Put reference back on failure
+          refCount.incrementAndGet();
+        }
+      }
     }
-    refCount--;
   }
   
   protected IndexReader() { 
-    refCount = 1;
+    refCount.set(1);
   }
   
   /**
    * @throws AlreadyClosedException if this IndexReader is closed
    */
   protected final void ensureOpen() throws AlreadyClosedException {
-    if (refCount <= 0) {
+    if (refCount.get() <= 0) {
       throw new AlreadyClosedException("this IndexReader is closed");
     }
   }
@@ -1362,4 +1390,13 @@
     }
     return numTerms;
   }
+
+  /** For IndexReader implementations that use
+   *  TermInfosReader to read terms, this returns the
+   *  current indexDivisor as specified when the reader was
+   *  opened.
+   */
+  public int getTermInfosIndexDivisor() {
+    throw new UnsupportedOperationException("This reader does not support this method.");
+  }
 }

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/IndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/IndexWriter.java?rev=899359&r1=899358&r2=899359&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/IndexWriter.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/IndexWriter.java Thu Jan 14 19:05:12 2010
@@ -388,6 +388,9 @@
    *  loading a TermInfo.  The default value is 1.  Set this
    *  to -1 to skip loading the terms index entirely. */
   public IndexReader getReader(int termInfosIndexDivisor) throws IOException {
+
+    ensureOpen();
+
     if (infoStream != null) {
       message("flush at getReader");
     }
@@ -397,12 +400,13 @@
     // this method is called:
     poolReaders = true;
 
-    flush(true, true, true);
+    flush(true, true, false);
     
     // Prevent segmentInfos from changing while opening the
     // reader; in theory we could do similar retry logic,
     // just like we do when loading segments_N
     synchronized(this) {
+      applyDeletes();
       return new ReadOnlyDirectoryReader(this, segmentInfos, termInfosIndexDivisor, codecs);
     }
   }
@@ -706,23 +710,19 @@
     notifyAll();
   }
 
-  synchronized final boolean isOpen(boolean includePendingClose) {
-    return !(closed || (includePendingClose && closing));
-  }
-
   /**
    * Used internally to throw an {@link
    * AlreadyClosedException} if this IndexWriter has been
    * closed.
    * @throws AlreadyClosedException if this IndexWriter is
    */
-  protected synchronized final void ensureOpen(boolean includePendingClose) throws AlreadyClosedException {
-    if (!isOpen(includePendingClose)) {
+  protected final void ensureOpen(boolean includePendingClose) throws AlreadyClosedException {
+    if (closed || (includePendingClose && closing)) {
       throw new AlreadyClosedException("this IndexWriter is closed");
     }
   }
 
-  protected synchronized final void ensureOpen() throws AlreadyClosedException {
+  protected final void ensureOpen() throws AlreadyClosedException {
     ensureOpen(true);
   }
 
@@ -1502,7 +1502,8 @@
             " maxBufferedDocs=" + docWriter.getMaxBufferedDocs() +
             " maxBuffereDeleteTerms=" + docWriter.getMaxBufferedDeleteTerms() +
             " maxFieldLength=" + maxFieldLength +
-            " index=" + segString());
+            " index=" + segString() +
+            " version=" + Constants.LUCENE_VERSION);
   }
 
   /**
@@ -2918,7 +2919,7 @@
     releaseWrite();
   }
 
-  private void blockAddIndexes(boolean includePendingClose) {
+  private void blockAddIndexes() {
 
     acquireRead();
 
@@ -2927,7 +2928,7 @@
 
       // Make sure we are still open since we could have
       // waited quite a while for last addIndexes to finish
-      ensureOpen(includePendingClose);
+      ensureOpen(false);
       success = true;
     } finally {
       if (!success)
@@ -3668,7 +3669,6 @@
       }
 
       if (flushDeletes) {
-        flushDeletesCount++;
         applyDeletes();
       }
       
@@ -3962,7 +3962,7 @@
       handleOOM(oom, "merge");
     }
     if (infoStream != null) {
-      message("merge time " + (System.currentTimeMillis()-t0) + " msec");
+      message("merge time " + (System.currentTimeMillis()-t0) + " msec for " + merge.info.docCount + " docs");
     }
   }
 
@@ -4453,6 +4453,7 @@
   // Apply buffered deletes to all segments.
   private final synchronized boolean applyDeletes() throws CorruptIndexException, IOException {
     assert testPoint("startApplyDeletes");
+    flushDeletesCount++;
     SegmentInfos rollback = (SegmentInfos) segmentInfos.clone();
     boolean success = false;
     boolean changed;
@@ -4514,7 +4515,7 @@
         buffer.append(' ');
       }
       final SegmentInfo info = infos.info(i);
-      buffer.append(info.segString(directory));
+      buffer.append(info.toString(directory, 0));
       if (info.dir != directory)
         buffer.append("**");
     }
@@ -4618,7 +4619,7 @@
         // Wait for any running addIndexes to complete
         // first, then block any from running until we've
         // copied the segmentInfos we intend to sync:
-        blockAddIndexes(false);
+        blockAddIndexes();
 
         // On commit the segmentInfos must never
         // reference a segment in another directory:

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/MergePolicy.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/MergePolicy.java?rev=899359&r1=899358&r2=899359&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/MergePolicy.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/MergePolicy.java Thu Jan 14 19:05:12 2010
@@ -85,6 +85,7 @@
     final boolean useCompoundFile;
     boolean aborted;
     Throwable error;
+    boolean paused;
 
     public OneMerge(SegmentInfos segments, boolean useCompoundFile) {
       if (0 == segments.size())
@@ -110,6 +111,7 @@
      *  not be committed. */
     synchronized void abort() {
       aborted = true;
+      notifyAll();
     }
 
     /** Returns true if this merge was aborted. */
@@ -118,8 +120,34 @@
     }
 
     synchronized void checkAborted(Directory dir) throws MergeAbortedException {
-      if (aborted)
+      if (aborted) {
         throw new MergeAbortedException("merge is aborted: " + segString(dir));
+      }
+
+      while (paused) {
+        try {
+          // In theory we could wait() indefinitely, but we
+          // do 1000 msec, defensively
+          wait(1000);
+        } catch (InterruptedException ie) {
+          throw new RuntimeException(ie);
+        }
+        if (aborted) {
+          throw new MergeAbortedException("merge is aborted: " + segString(dir));
+        }
+      }
+    }
+
+    synchronized public void setPause(boolean paused) {
+      this.paused = paused;
+      if (!paused) {
+        // Wakeup merge thread, if it's waiting
+        notifyAll();
+      }
+    }
+
+    synchronized public boolean getPause() {
+      return paused;
     }
 
     String segString(Directory dir) {
@@ -127,7 +155,7 @@
       final int numSegments = segments.size();
       for(int i=0;i<numSegments;i++) {
         if (i > 0) b.append(' ');
-        b.append(segments.info(i).segString(dir));
+        b.append(segments.info(i).toString(dir, 0));
       }
       if (info != null)
         b.append(" into ").append(info.name);
@@ -262,5 +290,4 @@
    * compound file format.
    */
   public abstract boolean useCompoundDocStore(SegmentInfos segments);
-  
 }

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/MultiReader.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/MultiReader.java?rev=899359&r1=899358&r2=899359&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/MultiReader.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/MultiReader.java Thu Jan 14 19:05:12 2010
@@ -32,6 +32,7 @@
 import org.apache.lucene.index.DirectoryReader.MultiTermPositions;
 import org.apache.lucene.search.Similarity;
 import org.apache.lucene.util.Bits;
+import org.apache.lucene.search.FieldCache; // not great (circular); used only to purge FieldCache entry on close
 
 /** An IndexReader which reads multiple indexes, appending
  * their content. */
@@ -257,8 +258,10 @@
   }
   
   @Override
-  public synchronized int numDocs() {
+  public int numDocs() {
     // Don't call ensureOpen() here (it could affect performance)
+    // NOTE: multiple threads may wind up init'ing
+    // numDocs... but that's harmless
     if (numDocs == -1) {        // check cache
       int n = 0;                // cache miss--recompute
       for (int i = 0; i < subReaders.length; i++)
@@ -459,6 +462,11 @@
         subReaders[i].close();
       }
     }
+
+    // NOTE: only needed in case someone had asked for
+    // FieldCache for top-level reader (which is generally
+    // not a good idea):
+    FieldCache.DEFAULT.purge(this);
   }
   
   @Override

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/ParallelReader.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/ParallelReader.java?rev=899359&r1=899358&r2=899359&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/ParallelReader.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/ParallelReader.java Thu Jan 14 19:05:12 2010
@@ -22,6 +22,7 @@
 import org.apache.lucene.document.FieldSelectorResult;
 import org.apache.lucene.document.Fieldable;
 import org.apache.lucene.util.Bits;
+import org.apache.lucene.search.FieldCache; // not great (circular); used only to purge FieldCache entry on close
 
 import java.io.IOException;
 import java.util.*;
@@ -72,6 +73,21 @@
     this.incRefReaders = !closeSubReaders;
   }
 
+  /** {@inheritDoc} */
+  @Override
+  public String toString() {
+    final StringBuilder buffer = new StringBuilder("ParallelReader(");
+    final Iterator<IndexReader> iter = readers.iterator();
+    if (iter.hasNext()) {
+      buffer.append(iter.next());
+    }
+    while (iter.hasNext()) {
+      buffer.append(", ").append(iter.next());
+    }
+    buffer.append(')');
+    return buffer.toString();
+  }
+
  /** Add an IndexReader.
   * @throws IOException if there is a low-level IO error
   */
@@ -544,6 +560,8 @@
         readers.get(i).close();
       }
     }
+
+    FieldCache.DEFAULT.purge(this);
   }
 
   @Override

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/SegmentInfo.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/SegmentInfo.java?rev=899359&r1=899358&r2=899359&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/SegmentInfo.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/SegmentInfo.java Thu Jan 14 19:05:12 2010
@@ -98,11 +98,6 @@
 
   private Map<String,String> diagnostics;
 
-  @Override
-  public String toString() {
-    return "si: "+dir.toString()+" "+name+" docCount: "+docCount+" delCount: "+delCount+" delFileName: "+getDelFileName();
-  }
-  
   public SegmentInfo(String name, int docCount, Directory dir, Codec codec) {
     this.name = name;
     this.docCount = docCount;
@@ -730,29 +725,68 @@
     sizeInBytes = -1;
   }
 
-  /** Used for debugging */
-  public String segString(Directory dir) {
-    String cfs;
+  /** {@inheritDoc} */
+  @Override
+  public String toString() {
+    return toString(dir, 0);
+  }
+
+  /** Used for debugging.  Format may suddenly change.
+   * 
+   *  <p>Current format looks like
+   *  <code>_a:c45/4->_1</code>, which means the segment's
+   *  name is <code>_a</code>; it's using compound file
+   *  format (would be <code>C</code> if not compound); it
+   *  has 45 documents; it has 4 deletions (this part is
+   *  left off when there are no deletions); it's using the
+   *  shared doc stores named <code>_1</code> (this part is
+   *  left off if doc stores are private).</p>
+   */
+  public String toString(Directory dir, int pendingDelCount) {
+
+    StringBuilder s = new StringBuilder();
+    s.append(name).append(':');
+
+    char cfs;
     try {
-      if (getUseCompoundFile())
-        cfs = "c";
-      else
-        cfs = "C";
+      if (getUseCompoundFile()) {
+        cfs = 'c';
+      } else {
+        cfs = 'C';
+      }
     } catch (IOException ioe) {
-      cfs = "?";
+      cfs = '?';
     }
+    s.append(cfs);
 
-    String docStore;
+    if (this.dir != dir) {
+      s.append('x');
+    }
+    s.append(docCount);
 
-    if (docStoreOffset != -1)
-      docStore = "->" + docStoreSegment;
-    else
-      docStore = "";
+    int delCount;
+    try {
+      delCount = getDelCount();
+    } catch (IOException ioe) {
+      delCount = -1;
+    }
+    if (delCount != -1) {
+      delCount += pendingDelCount;
+    }
+    if (delCount != 0) {
+      s.append('/');
+      if (delCount == -1) {
+        s.append('?');
+      } else {
+        s.append(delCount);
+      }
+    }
+    
+    if (docStoreOffset != -1) {
+      s.append("->").append(docStoreSegment);
+    }
 
-    return name + ":" +
-      cfs +
-      (this.dir == dir ? "" : "x") +
-      docCount + docStore;
+    return s.toString();
   }
 
   /** We consider another SegmentInfo instance equal if it

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/SegmentInfos.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/SegmentInfos.java?rev=899359&r1=899358&r2=899359&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/SegmentInfos.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/SegmentInfos.java Thu Jan 14 19:05:12 2010
@@ -878,17 +878,16 @@
     finishCommit(dir);
   }
 
-  public synchronized String segString(Directory directory) {
+  public synchronized String toString(Directory directory) {
     StringBuilder buffer = new StringBuilder();
+    buffer.append(getCurrentSegmentFileName()).append(": ");
     final int count = size();
     for(int i = 0; i < count; i++) {
       if (i > 0) {
         buffer.append(' ');
       }
       final SegmentInfo info = info(i);
-      buffer.append(info.segString(directory));
-      if (info.dir != directory)
-        buffer.append("**");
+      buffer.append(info.toString(directory, 0));
     }
     return buffer.toString();
   }
@@ -923,4 +922,14 @@
         return true;
     return false;
   }
+
+  /** Returns sum of all segment's docCounts.  Note that
+   *  this does not include deletions */
+  public int totalDocCount() {
+    int count = 0;
+    for(SegmentInfo info : this) {
+      count += info.docCount;
+    }
+    return count;
+  }
 }

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/SegmentReader.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/SegmentReader.java?rev=899359&r1=899358&r2=899359&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/SegmentReader.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/SegmentReader.java Thu Jan 14 19:05:12 2010
@@ -46,6 +46,7 @@
 import org.apache.lucene.index.codecs.preflex.SegmentTermDocs;
 import org.apache.lucene.index.codecs.preflex.SegmentTermPositions;
 import org.apache.lucene.index.codecs.FieldsProducer;
+import org.apache.lucene.search.FieldCache; // not great (circular); used only to purge FieldCache entry on close
 
 /** @version $Id */
 /**
@@ -103,12 +104,14 @@
     final int readBufferSize;
     final int termsIndexDivisor;
 
+    private final SegmentReader origInstance;
+
     FieldsReader fieldsReaderOrig;
     TermVectorsReader termVectorsReaderOrig;
     CompoundFileReader cfsReader;
     CompoundFileReader storeCFSReader;
 
-    CoreReaders(Directory dir, SegmentInfo si, int readBufferSize, int termsIndexDivisor, Codecs codecs) throws IOException {
+    CoreReaders(SegmentReader origInstance, Directory dir, SegmentInfo si, int readBufferSize, int termsIndexDivisor, Codecs codecs) throws IOException {
 
       if (termsIndexDivisor < 1 && termsIndexDivisor != -1) {
         throw new IllegalArgumentException("indexDivisor must be -1 (don't load terms index) or greater than 0: got " + termsIndexDivisor);
@@ -153,6 +156,12 @@
           decRef();
         }
       }
+
+      // Must assign this at the end -- if we hit an
+      // exception above core, we don't want to attempt to
+      // purge the FieldCache (will hit NPE because core is
+      // not assigned yet).
+      this.origInstance = origInstance;
     }
 
     synchronized TermVectorsReader getTermVectorsReaderOrig() {
@@ -194,6 +203,11 @@
         if (storeCFSReader != null) {
           storeCFSReader.close();
         }
+
+        // Force FieldCache to evict our entries at this point
+        if (origInstance != null) {
+          FieldCache.DEFAULT.purge(origInstance);
+        }
       }
     }
 
@@ -515,7 +529,7 @@
     boolean success = false;
 
     try {
-      instance.core = new CoreReaders(dir, si, readBufferSize, termInfosIndexDivisor, codecs);
+      instance.core = new CoreReaders(instance, dir, si, readBufferSize, termInfosIndexDivisor, codecs);
       if (doOpenStores) {
         instance.core.openDocStores(si);
       }
@@ -545,20 +559,31 @@
     return deletedDocs;
   }
 
+  private boolean checkDeletedCounts() throws IOException {
+    final int recomputedCount = deletedDocs.getRecomputedCount();
+     
+    assert deletedDocs.count() == recomputedCount : "deleted count=" + deletedDocs.count() + " vs recomputed count=" + recomputedCount;
+
+    assert si.getDelCount() == recomputedCount : 
+    "delete count mismatch: info=" + si.getDelCount() + " vs BitVector=" + recomputedCount;
+
+    // Verify # deletes does not exceed maxDoc for this
+    // segment:
+    assert si.getDelCount() <= maxDoc() : 
+    "delete count mismatch: " + recomputedCount + ") exceeds max doc (" + maxDoc() + ") for segment " + si.name;
+
+    return true;
+  }
+
   private void loadDeletedDocs() throws IOException {
     // NOTE: the bitvector is stored using the regular directory, not cfs
     if (hasDeletions(si)) {
       deletedDocs = new BitVector(directory(), si.getDelFileName());
       deletedDocsRef = new AtomicInteger(1);
-     
-      assert si.getDelCount() == deletedDocs.count() : 
-        "delete count mismatch: info=" + si.getDelCount() + " vs BitVector=" + deletedDocs.count();
-
-      // Verify # deletes does not exceed maxDoc for this
-      // segment:
-      assert si.getDelCount() <= maxDoc() : 
-        "delete count mismatch: " + deletedDocs.count() + ") exceeds max doc (" + maxDoc() + ") for segment " + si.name;
-
+      assert checkDeletedCounts();
+      if (deletedDocs.size() != si.docCount) {
+        throw new CorruptIndexException("document count mismatch: deleted docs count " + deletedDocs.size() + " vs segment doc count " + si.docCount);
+      }
     } else
       assert si.getDelCount() == 0;
   }
@@ -631,10 +656,10 @@
       clone.readOnly = openReadOnly;
       clone.si = si;
       clone.readBufferSize = readBufferSize;
+      clone.pendingDeleteCount = pendingDeleteCount;
 
       if (!openReadOnly && hasChanges) {
         // My pending changes transfer to the new reader
-        clone.pendingDeleteCount = pendingDeleteCount;
         clone.deletedDocsDirty = deletedDocsDirty;
         clone.normsDirty = normsDirty;
         clone.hasChanges = hasChanges;
@@ -1206,6 +1231,17 @@
     return termVectorsReader.get(docNumber);
   }
   
+  /** {@inheritDoc} */
+  @Override
+  public String toString() {
+    final StringBuilder buffer = new StringBuilder();
+    if (hasChanges) {
+      buffer.append('*');
+    }
+    buffer.append(si.toString(core.dir, pendingDeleteCount));
+    return buffer.toString();
+  }
+
   /**
    * Return the name of the segment this reader is reading.
    */
@@ -1276,6 +1312,7 @@
    * We do it with R/W access for the tests (BW compatibility)
    * @deprecated Remove this when tests are fixed!
    */
+  @Deprecated
   static SegmentReader getOnlySegmentReader(Directory dir) throws IOException {
     return getOnlySegmentReader(IndexReader.open(dir, false));
   }
@@ -1295,6 +1332,7 @@
     throw new IllegalArgumentException(reader + " is not a SegmentReader or a single-segment DirectoryReader");
   }
 
+  @Override
   public int getTermInfosIndexDivisor() {
     return core.termsIndexDivisor;
   }

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/queryParser/MultiFieldQueryParser.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/queryParser/MultiFieldQueryParser.java?rev=899359&r1=899358&r2=899359&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/queryParser/MultiFieldQueryParser.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/queryParser/MultiFieldQueryParser.java Thu Jan 14 19:05:12 2010
@@ -96,6 +96,7 @@
     this.fields = fields;
   }
   
+  @Override
   protected Query getFieldQuery(String field, String queryText, int slop) throws ParseException {
     if (field == null) {
       List<BooleanClause> clauses = new ArrayList<BooleanClause>();
@@ -132,11 +133,13 @@
   }
   
 
+  @Override
   protected Query getFieldQuery(String field, String queryText) throws ParseException {
     return getFieldQuery(field, queryText, 0);
   }
 
 
+  @Override
   protected Query getFuzzyQuery(String field, String termStr, float minSimilarity) throws ParseException
   {
     if (field == null) {
@@ -150,6 +153,7 @@
     return super.getFuzzyQuery(field, termStr, minSimilarity);
   }
 
+  @Override
   protected Query getPrefixQuery(String field, String termStr) throws ParseException
   {
     if (field == null) {
@@ -163,6 +167,7 @@
     return super.getPrefixQuery(field, termStr);
   }
 
+  @Override
   protected Query getWildcardQuery(String field, String termStr) throws ParseException {
     if (field == null) {
       List<BooleanClause> clauses = new ArrayList<BooleanClause>();
@@ -176,6 +181,7 @@
   }
 
  
+  @Override
   protected Query getRangeQuery(String field, String part1, String part2, boolean inclusive) throws ParseException {
     if (field == null) {
       List<BooleanClause> clauses = new ArrayList<BooleanClause>();

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/FieldCache.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/FieldCache.java?rev=899359&r1=899358&r2=899359&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/FieldCache.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/FieldCache.java Thu Jan 14 19:05:12 2010
@@ -614,6 +614,15 @@
   public abstract void purgeAllCaches();
 
   /**
+   * Expert: drops all cache entries associated with this
+   * reader.  NOTE: this reader must precisely match the
+   * reader that the cache entry is keyed on. If you pass a
+   * top-level reader, it usually will have no effect as
+   * Lucene now caches at the segment reader level.
+   */
+  public abstract void purge(IndexReader r);
+
+  /**
    * If non-null, FieldCacheImpl will warn whenever
    * entries are created that are not sane according to
    * {@link org.apache.lucene.util.FieldCacheSanityChecker}.

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/FieldCacheImpl.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/FieldCacheImpl.java?rev=899359&r1=899358&r2=899359&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/FieldCacheImpl.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/FieldCacheImpl.java Thu Jan 14 19:05:12 2010
@@ -65,6 +65,12 @@
   public void purgeAllCaches() {
     init();
   }
+
+  public void purge(IndexReader r) {
+    for(Cache c : caches.values()) {
+      c.purge(r);
+    }
+  }
   
   public CacheEntry[] getCacheEntries() {
     List<CacheEntry> result = new ArrayList<CacheEntry>(17);
@@ -148,6 +154,14 @@
     protected abstract Object createValue(IndexReader reader, Entry key)
         throws IOException;
 
+    /** Remove this reader from the cache, if present. */
+    public void purge(IndexReader r) {
+      Object readerKey = r.getFieldCacheKey();
+      synchronized(readerCache) {
+        readerCache.remove(readerKey);
+      }
+    }
+
     public Object get(IndexReader reader, Entry key) throws IOException {
       Map<Entry,Object> innerCache;
       Object value;

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/SortField.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/SortField.java?rev=899359&r1=899358&r2=899359&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/SortField.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/SortField.java Thu Jan 14 19:05:12 2010
@@ -21,10 +21,6 @@
 import java.io.Serializable;
 import java.util.Locale;
 
-import org.apache.lucene.document.NumericField; // javadocs
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.index.TermEnum;
 import org.apache.lucene.util.StringHelper;
 
 /**
@@ -258,6 +254,13 @@
     return reverse;
   }
 
+  /** Returns the {@link FieldComparatorSource} used for
+   * custom sorting
+   */
+  public FieldComparatorSource getComparatorSource() {
+    return comparatorSource;
+  }
+
   @Override
   public String toString() {
     StringBuilder buffer = new StringBuilder();

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/function/CustomScoreQuery.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/function/CustomScoreQuery.java?rev=899359&r1=899358&r2=899359&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/function/CustomScoreQuery.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/function/CustomScoreQuery.java Thu Jan 14 19:05:12 2010
@@ -207,6 +207,9 @@
    * <pre>
    *     ModifiedScore = subQueryScore * valSrcScore
    * </pre>
+   *
+   * <p><b>NOTE</b>: The doc is relative to the current
+   * reader, last passed to {@link #setNextReader}.
    * 
    * @param doc id of scored doc. 
    * @param subQueryScore score of that doc by the subQuery.
@@ -218,6 +221,15 @@
   }
 
   /**
+   * Called when the scoring switches to another reader.
+   * 
+   * @param reader
+   *          next IndexReader
+   */
+  public void setNextReader(IndexReader reader) throws IOException {
+  }
+
+  /**
    * Explain the custom score.
    * Whenever overriding {@link #customScore(int, float, float[])}, 
    * this method should also be overridden to provide the correct explanation
@@ -385,7 +397,6 @@
    * A scorer that applies a (callback) function on scores of the subQuery.
    */
   private class CustomScorer extends Scorer {
-    private final CustomWeight weight;
     private final float qWeight;
     private Scorer subQueryScorer;
     private Scorer[] valSrcScorers;
@@ -396,12 +407,12 @@
     private CustomScorer(Similarity similarity, IndexReader reader, CustomWeight w,
         Scorer subQueryScorer, Scorer[] valSrcScorers) throws IOException {
       super(similarity);
-      this.weight = w;
       this.qWeight = w.getValue();
       this.subQueryScorer = subQueryScorer;
       this.valSrcScorers = valSrcScorers;
       this.reader = reader;
       this.vScores = new float[valSrcScorers.length];
+      setNextReader(reader);
     }
 
     @Override

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/spans/FieldMaskingSpanQuery.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/spans/FieldMaskingSpanQuery.java?rev=899359&r1=899358&r2=899359&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/spans/FieldMaskingSpanQuery.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/spans/FieldMaskingSpanQuery.java Thu Jan 14 19:05:12 2010
@@ -59,7 +59,7 @@
  * <pre>
  *    SpanQuery q1  = new SpanTermQuery(new Term("studentfirstname", "james"));
  *    SpanQuery q2  = new SpanTermQuery(new Term("studentsurname", "jones"));
- *    SpanQuery q2m new FieldMaskingSpanQuery(q2, "studentfirstname");
+ *    SpanQuery q2m = new FieldMaskingSpanQuery(q2, "studentfirstname");
  *    Query q = new SpanNearQuery(new SpanQuery[]{q1, q2m}, -1, false);
  * </pre>
  * to search for 'studentfirstname:james studentsurname:jones' and find 

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/store/FSDirectory.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/store/FSDirectory.java?rev=899359&r1=899358&r2=899359&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/store/FSDirectory.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/store/FSDirectory.java Thu Jan 14 19:05:12 2010
@@ -373,6 +373,7 @@
   }
 
   /** @deprecated Use {@link #getDirectory} instead. */
+  @Deprecated
   public File getFile() {
     return getDirectory();
   }

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/store/IndexInput.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/store/IndexInput.java?rev=899359&r1=899358&r2=899359&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/store/IndexInput.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/store/IndexInput.java Thu Jan 14 19:05:12 2010
@@ -146,6 +146,7 @@
    *                instead, and construct the string
    *                from those utf8 bytes
    */
+  @Deprecated
   public void readChars(char[] buffer, int start, int length)
        throws IOException {
     final int end = start + length;
@@ -174,6 +175,7 @@
    * @deprecated this method operates on old "modified utf8" encoded
    *             strings
    */
+  @Deprecated
   public void skipChars(int length) throws IOException{
     for (int i = 0; i < length; i++) {
       byte b = readByte();

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/store/IndexOutput.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/store/IndexOutput.java?rev=899359&r1=899358&r2=899359&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/store/IndexOutput.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/store/IndexOutput.java Thu Jan 14 19:05:12 2010
@@ -114,6 +114,7 @@
    * @deprecated -- please pre-convert to utf8 bytes
    * instead or use {@link #writeString}
    */
+  @Deprecated
   public void writeChars(String s, int start, int length)
        throws IOException {
     final int end = start + length;
@@ -139,6 +140,7 @@
    * @param length the number of characters in the sequence
    * @deprecated -- please pre-convert to utf8 bytes instead or use {@link #writeString}
    */
+  @Deprecated
   public void writeChars(char[] s, int start, int length)
     throws IOException {
     final int end = start + length;

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/store/MMapDirectory.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/store/MMapDirectory.java?rev=899359&r1=899358&r2=899359&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/store/MMapDirectory.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/store/MMapDirectory.java Thu Jan 14 19:05:12 2010
@@ -401,7 +401,7 @@
   @Override
   public IndexInput openInput(String name, int bufferSize) throws IOException {
     ensureOpen();
-    File f =  new File(getFile(), name);
+    File f =  new File(getDirectory(), name);
     RandomAccessFile raf = new RandomAccessFile(f, "r");
     try {
       return (raf.length() <= (long) maxBBuf)

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/store/NIOFSDirectory.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/store/NIOFSDirectory.java?rev=899359&r1=899358&r2=899359&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/store/NIOFSDirectory.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/store/NIOFSDirectory.java Thu Jan 14 19:05:12 2010
@@ -64,7 +64,7 @@
   @Override
   public IndexInput openInput(String name, int bufferSize) throws IOException {
     ensureOpen();
-    return new NIOFSIndexInput(new File(getFile(), name), bufferSize, getReadChunkSize());
+    return new NIOFSIndexInput(new File(getDirectory(), name), bufferSize, getReadChunkSize());
   }
 
   /** Creates an IndexOutput for the file with the given name. */



Mime
View raw message