lucene-java-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From markrmil...@apache.org
Subject svn commit: r880963 [2/7] - in /lucene/java/branches/flex_1458: ./ contrib/ contrib/analyzers/common/src/java/org/apache/lucene/analysis/ar/ contrib/analyzers/common/src/java/org/apache/lucene/analysis/br/ contrib/analyzers/common/src/java/org/apache/l...
Date Mon, 16 Nov 2009 21:24:47 GMT
Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/query/QueryAutoStopWordAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/query/QueryAutoStopWordAnalyzer.java?rev=880963&r1=880962&r2=880963&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/query/QueryAutoStopWordAnalyzer.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/query/QueryAutoStopWordAnalyzer.java Mon Nov 16 21:24:41 2009
@@ -18,9 +18,7 @@
 
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.Term;
-import org.apache.lucene.index.TermsEnum;
-import org.apache.lucene.index.Terms;
-import org.apache.lucene.index.TermRef;
+import org.apache.lucene.index.TermEnum;
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.StopFilter;
@@ -47,7 +45,7 @@
  */
 public class QueryAutoStopWordAnalyzer extends Analyzer {
   Analyzer delegate;
-  HashMap stopWordsPerField = new HashMap();
+  HashMap<String,HashSet<String>> stopWordsPerField = new HashMap<String,HashSet<String>>();
   //The default maximum percentage (40%) of index documents which
   //can contain a term, after which the term is considered to be a stop word.
   public static final float defaultMaxDocFreqPercent = 0.4f;
@@ -88,9 +86,9 @@
    */
   public int addStopWords(IndexReader reader, int maxDocFreq) throws IOException {
     int numStopWords = 0;
-    Collection fieldNames = reader.getFieldNames(IndexReader.FieldOption.INDEXED);
-    for (Iterator iter = fieldNames.iterator(); iter.hasNext();) {
-      String fieldName = (String) iter.next();
+    Collection<String> fieldNames = reader.getFieldNames(IndexReader.FieldOption.INDEXED);
+    for (Iterator<String> iter = fieldNames.iterator(); iter.hasNext();) {
+      String fieldName = iter.next();
       numStopWords += addStopWords(reader, fieldName, maxDocFreq);
     }
     return numStopWords;
@@ -108,9 +106,9 @@
    */
   public int addStopWords(IndexReader reader, float maxPercentDocs) throws IOException {
     int numStopWords = 0;
-    Collection fieldNames = reader.getFieldNames(IndexReader.FieldOption.INDEXED);
-    for (Iterator iter = fieldNames.iterator(); iter.hasNext();) {
-      String fieldName = (String) iter.next();
+    Collection<String> fieldNames = reader.getFieldNames(IndexReader.FieldOption.INDEXED);
+    for (Iterator<String> iter = fieldNames.iterator(); iter.hasNext();) {
+      String fieldName = iter.next();
       numStopWords += addStopWords(reader, fieldName, maxPercentDocs);
     }
     return numStopWords;
@@ -143,29 +141,28 @@
    * @throws IOException
    */
   public int addStopWords(IndexReader reader, String fieldName, int maxDocFreq) throws IOException {
-    HashSet stopWords = new HashSet();
+    HashSet<String> stopWords = new HashSet<String>();
     String internedFieldName = StringHelper.intern(fieldName);
-    Terms terms = reader.fields().terms(fieldName);
-    if (terms != null) {
-      TermsEnum termsEnum = terms.iterator();
-      while(true) {
-        TermRef text = termsEnum.next();
-        if (text != null) {
-          if (termsEnum.docFreq() > maxDocFreq) {
-            stopWords.add(text.toString());
-          }
-        } else {
-          break;
-        }
+    TermEnum te = reader.terms(new Term(fieldName));
+    Term term = te.term();
+    while (term != null) {
+      if (term.field() != internedFieldName) {
+        break;
       }
+      if (te.docFreq() > maxDocFreq) {
+        stopWords.add(term.text());
+      }
+      if (!te.next()) {
+        break;
+      }
+      term = te.term();
     }
-
     stopWordsPerField.put(fieldName, stopWords);
     
     /* if the stopwords for a field are changed,
      * then saved streams for that field are erased.
      */
-    Map streamMap = (Map) getPreviousTokenStream();
+    Map<String,SavedStreams> streamMap = (Map<String,SavedStreams>) getPreviousTokenStream();
     if (streamMap != null)
       streamMap.remove(fieldName);
     
@@ -180,7 +177,7 @@
     } catch (IOException e) {
       result = delegate.tokenStream(fieldName, reader);
     }
-    HashSet stopWords = (HashSet) stopWordsPerField.get(fieldName);
+    HashSet<String> stopWords = stopWordsPerField.get(fieldName);
     if (stopWords != null) {
       result = new StopFilter(StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion),
                               result, stopWords);
@@ -210,13 +207,13 @@
     }
 
     /* map of SavedStreams for each field */
-    Map streamMap = (Map) getPreviousTokenStream();
+    Map<String,SavedStreams> streamMap = (Map<String,SavedStreams>) getPreviousTokenStream();
     if (streamMap == null) {
-      streamMap = new HashMap();
+      streamMap = new HashMap<String, SavedStreams>();
       setPreviousTokenStream(streamMap);
     }
 
-    SavedStreams streams = (SavedStreams) streamMap.get(fieldName);
+    SavedStreams streams = streamMap.get(fieldName);
     if (streams == null) {
       /* an entry for this field does not exist, create one */
       streams = new SavedStreams();
@@ -224,7 +221,7 @@
       streams.wrapped = delegate.reusableTokenStream(fieldName, reader);
 
       /* if there are any stopwords for the field, save the stopfilter */
-      HashSet stopWords = (HashSet) stopWordsPerField.get(fieldName);
+      HashSet<String> stopWords = stopWordsPerField.get(fieldName);
       if (stopWords != null)
         streams.withStopFilter = new StopFilter(StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion),
                                                 streams.wrapped, stopWords);
@@ -246,7 +243,7 @@
          * field, create a new StopFilter around the new stream
          */
         streams.wrapped = result;
-        HashSet stopWords = (HashSet) stopWordsPerField.get(fieldName);
+        HashSet<String> stopWords = stopWordsPerField.get(fieldName);
         if (stopWords != null)
           streams.withStopFilter = new StopFilter(StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion),
                                                   streams.wrapped, stopWords);
@@ -267,9 +264,9 @@
    */
   public String[] getStopWords(String fieldName) {
     String[] result;
-    HashSet stopWords = (HashSet) stopWordsPerField.get(fieldName);
+    HashSet<String> stopWords = stopWordsPerField.get(fieldName);
     if (stopWords != null) {
-      result = (String[]) stopWords.toArray(new String[stopWords.size()]);
+      result = stopWords.toArray(new String[stopWords.size()]);
     } else {
       result = new String[0];
     }
@@ -282,16 +279,16 @@
    * @return the stop words (as terms)
    */
   public Term[] getStopWords() {
-    ArrayList allStopWords = new ArrayList();
-    for (Iterator iter = stopWordsPerField.keySet().iterator(); iter.hasNext();) {
-      String fieldName = (String) iter.next();
-      HashSet stopWords = (HashSet) stopWordsPerField.get(fieldName);
-      for (Iterator iterator = stopWords.iterator(); iterator.hasNext();) {
-        String text = (String) iterator.next();
+    ArrayList<Term> allStopWords = new ArrayList<Term>();
+    for (Iterator<String> iter = stopWordsPerField.keySet().iterator(); iter.hasNext();) {
+      String fieldName = iter.next();
+      HashSet<String> stopWords = stopWordsPerField.get(fieldName);
+      for (Iterator<String> iterator = stopWords.iterator(); iterator.hasNext();) {
+        String text = iterator.next();
         allStopWords.add(new Term(fieldName, text));
       }
     }
-    return (Term[]) allStopWords.toArray(new Term[allStopWords.size()]);
+    return allStopWords.toArray(new Term[allStopWords.size()]);
 	}
 
 }

Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ru/RussianAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ru/RussianAnalyzer.java?rev=880963&r1=880962&r2=880963&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ru/RussianAnalyzer.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ru/RussianAnalyzer.java Mon Nov 16 21:24:41 2009
@@ -19,11 +19,12 @@
 
 import java.io.IOException;
 import java.io.Reader;
-import java.util.HashSet;
+import java.util.Arrays;
 import java.util.Map;
 import java.util.Set;
 
 import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.CharArraySet;
 import org.apache.lucene.analysis.LowerCaseFilter;
 import org.apache.lucene.analysis.StopFilter;
 import org.apache.lucene.analysis.TokenStream;
@@ -55,37 +56,53 @@
       "тоже", "той", "только", "том", "ты", "у", "уже", "хотя", "чего", "чей", 
       "чем", "что", "чтобы", "чье", "чья", "эта", "эти", "это", "я"
     };
+    
+    private static class DefaultSetHolder {
+      static final Set<?> DEFAULT_STOP_SET = CharArraySet
+          .unmodifiableSet(new CharArraySet(Arrays.asList(RUSSIAN_STOP_WORDS),
+              false));
+    }
 
     /**
      * Contains the stopwords used with the StopFilter.
      */
-    private Set stopSet = new HashSet();
+    private final Set<?> stopSet;
 
     private final Version matchVersion;
 
     public RussianAnalyzer(Version matchVersion) {
-      this(matchVersion, RUSSIAN_STOP_WORDS);
+      this(matchVersion, DefaultSetHolder.DEFAULT_STOP_SET);
     }
   
     /**
      * Builds an analyzer with the given stop words.
+     * @deprecated use {@link #RussianAnalyzer(Version, Set)} instead
      */
-    public RussianAnalyzer(Version matchVersion, String... stopwords)
-    {
-      super();
-      stopSet = StopFilter.makeStopSet(stopwords);
+    public RussianAnalyzer(Version matchVersion, String... stopwords) {
+      this(matchVersion, StopFilter.makeStopSet(stopwords));
+    }
+    
+    /**
+     * Builds an analyzer with the given stop words
+     * 
+     * @param matchVersion
+     *          lucene compatibility version
+     * @param stopwords
+     *          a stopword set
+     */
+    public RussianAnalyzer(Version matchVersion, Set<?> stopwords){
+      stopSet = CharArraySet.unmodifiableSet(CharArraySet.copy(stopwords));
       this.matchVersion = matchVersion;
     }
    
     /**
      * Builds an analyzer with the given stop words.
      * TODO: create a Set version of this ctor
+     * @deprecated use {@link #RussianAnalyzer(Version, Set)} instead
      */
-    public RussianAnalyzer(Version matchVersion, Map stopwords)
+    public RussianAnalyzer(Version matchVersion, Map<?,?> stopwords)
     {
-      super();
-      stopSet = new HashSet(stopwords.keySet());
-      this.matchVersion = matchVersion;
+      this(matchVersion, stopwords.keySet());
     }
 
     /**

Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/shingle/ShingleFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/shingle/ShingleFilter.java?rev=880963&r1=880962&r2=880963&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/shingle/ShingleFilter.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/shingle/ShingleFilter.java Mon Nov 16 21:24:41 2009
@@ -29,6 +29,7 @@
 import org.apache.lucene.analysis.tokenattributes.TermAttribute;
 import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
 import org.apache.lucene.util.AttributeSource;
+import org.apache.lucene.util.AttributeSource.State;
 
 /**
  * <p>A ShingleFilter constructs shingles (token n-grams) from a token stream.
@@ -43,7 +44,7 @@
  */
 public final class ShingleFilter extends TokenFilter {
 
-  private LinkedList shingleBuf = new LinkedList();
+  private LinkedList<State> shingleBuf = new LinkedList<State>();
   private StringBuilder[] shingles;
   private String tokenType = "shingle";
 
@@ -172,7 +173,7 @@
         }
       }
       
-      nextToken = (AttributeSource.State) shingleBuf.getFirst();
+      nextToken = shingleBuf.getFirst();
       
       if (outputUnigrams) {
         if (shingleBufferPosition == 0) {
@@ -313,8 +314,8 @@
     }
 
     int i = 0;
-    for (Iterator it = shingleBuf.iterator(); it.hasNext(); ) {
-      restoreState((AttributeSource.State) it.next());
+    for (Iterator<State> it = shingleBuf.iterator(); it.hasNext(); ) {
+      restoreState(it.next());
       for (int j = i; j < shingles.length; j++) {
         if (shingles[j].length() != 0) {
           shingles[j].append(TOKEN_SEPARATOR);

Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/shingle/ShingleMatrixFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/shingle/ShingleMatrixFilter.java?rev=880963&r1=880962&r2=880963&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/shingle/ShingleMatrixFilter.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/shingle/ShingleMatrixFilter.java Mon Nov 16 21:24:41 2009
@@ -30,6 +30,8 @@
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.miscellaneous.EmptyTokenStream;
 import org.apache.lucene.analysis.payloads.PayloadHelper;
+import org.apache.lucene.analysis.shingle.ShingleMatrixFilter.Matrix.Column;
+import org.apache.lucene.analysis.shingle.ShingleMatrixFilter.Matrix.Column.Row;
 import org.apache.lucene.analysis.tokenattributes.FlagsAttribute;
 import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
 import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
@@ -213,7 +215,7 @@
    * The filter /will/ delete columns from the input matrix! You will not be able to reset the filter if you used this constructor.
    * todo: don't touch the matrix! use a boolean, set the input stream to null or something, and keep track of where in the matrix we are at.
    *
-   * @param matrix the input based for creating shingles. Does not need to contain any information until {@link org.apache.lucene.analysis.shingle.ShingleMatrixFilter#next(org.apache.lucene.analysis.Token)} is called the first time.
+   * @param matrix the input based for creating shingles. Does not need to contain any information until {@link #incrementToken()} is called the first time.
    * @param minimumShingleSize minimum number of tokens in any shingle.
    * @param maximumShingleSize maximum number of tokens in any shingle.
    * @param spacerCharacter character to use between texts of the token parts in a shingle. null for none.
@@ -339,7 +341,7 @@
   private int currentShingleLength;
 
   /**
-   * a set containing shingles that has been the result of a call to next(Token),
+   * a set containing shingles that has been the result of a call to {@link #incrementToken()},
    * used to avoid producing the same shingle more than once.
    */
   private Set<List<Token>> shinglesSeen = new HashSet<List<Token>>();
@@ -567,7 +569,7 @@
   }
 
   /**
-   * Final touch of a shingle token before it is passed on to the consumer from method {@link #next(org.apache.lucene.analysis.Token)}.
+   * Final touch of a shingle token before it is passed on to the consumer from method {@link #incrementToken()}.
    *
    * Calculates and sets type, flags, position increment, start/end offsets and weight.
    *
@@ -577,12 +579,12 @@
    * @param currentPermutationRows index to Matrix.Column.Row from the position of tokens in parameter currentPermutationTokens
    * @param currentPermuationTokens tokens of the current permutation of rows in the matrix.
    */
-  public void updateToken(Token token, List shingle, int currentPermutationStartOffset, List currentPermutationRows, List currentPermuationTokens) {
+  public void updateToken(Token token, List<Token> shingle, int currentPermutationStartOffset, List<Row> currentPermutationRows, List<Token> currentPermuationTokens) {
     token.setType(ShingleMatrixFilter.class.getName());
     token.setFlags(0);
     token.setPositionIncrement(1);
-    token.setStartOffset(((Token) shingle.get(0)).startOffset());
-    token.setEndOffset(((Token) shingle.get(shingle.size() - 1)).endOffset());
+    token.setStartOffset(shingle.get(0).startOffset());
+    token.setEndOffset(shingle.get(shingle.size() - 1).endOffset());
     settingsCodec.setWeight(token, calculateShingleWeight(token, shingle, currentPermutationStartOffset, currentPermutationRows, currentPermuationTokens));
   }
 
@@ -602,7 +604,7 @@
    * @param currentPermuationTokens all tokens in the current row permutation of the matrix. A sub list (parameter offset, parameter shingle.size) equals parameter shingle.
    * @return weight to be set for parameter shingleToken
    */
-  public float calculateShingleWeight(Token shingleToken, List shingle, int currentPermutationStartOffset, List currentPermutationRows, List currentPermuationTokens) {
+  public float calculateShingleWeight(Token shingleToken, List<Token> shingle, int currentPermutationStartOffset, List<Row> currentPermutationRows, List<Token> currentPermuationTokens) {
     double[] weights = new double[shingle.size()];
 
     double total = 0f;
@@ -610,7 +612,7 @@
 
 
     for (int i=0; i<weights.length; i++) {
-      weights[i] = settingsCodec.getWeight((Token) shingle.get(i));
+      weights[i] = settingsCodec.getWeight(shingle.get(i));
 
       double tmp = weights[i];
       if (tmp > top) {
@@ -705,7 +707,7 @@
 
     private List<Column> columns = new ArrayList<Column>();
 
-    public List getColumns() {
+    public List<Column> getColumns() {
       return columns;
     }
 

Modified: lucene/java/branches/flex_1458/contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/BiSegGraph.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/BiSegGraph.java?rev=880963&r1=880962&r2=880963&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/BiSegGraph.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/BiSegGraph.java Mon Nov 16 21:24:41 2009
@@ -206,7 +206,7 @@
 
     rpath.add(current);
     while (current != 0) {
-      PathNode currentPathNode = (PathNode) path.get(current);
+      PathNode currentPathNode = path.get(current);
       preNode = currentPathNode.preNode;
       rpath.add(Integer.valueOf(preNode));
       current = preNode;

Modified: lucene/java/branches/flex_1458/contrib/ant/src/java/org/apache/lucene/ant/IndexTask.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/ant/src/java/org/apache/lucene/ant/IndexTask.java?rev=880963&r1=880962&r2=880963&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/ant/src/java/org/apache/lucene/ant/IndexTask.java (original)
+++ lucene/java/branches/flex_1458/contrib/ant/src/java/org/apache/lucene/ant/IndexTask.java Mon Nov 16 21:24:41 2009
@@ -63,7 +63,7 @@
   /**
    *  resources
    */
-  protected Vector rcs = new Vector();
+  protected Vector<ResourceCollection> rcs = new Vector<ResourceCollection>();
 
   /**
    *  overwrite index?
@@ -225,10 +225,9 @@
 
     // construct handler and analyzer dynamically
     try {
-      Class clazz = Class.forName(handlerClassName);
-      handler = (DocumentHandler) clazz.newInstance();
+      handler = Class.forName(handlerClassName).asSubclass(DocumentHandler.class).newInstance();
 
-      analyzer = this.createAnalyzer(analyzerClassName);
+      analyzer = IndexTask.createAnalyzer(analyzerClassName);
     } catch (Exception e) {
       throw new BuildException(e);
     }
@@ -292,7 +291,7 @@
         writer.setMergeFactor(mergeFactor);
 
         for (int i = 0; i < rcs.size(); i++) {
-          ResourceCollection rc = (ResourceCollection) rcs.elementAt(i);
+          ResourceCollection rc = rcs.elementAt(i);
           if (rc.isFilesystemOnly()) {
             Iterator resources = rc.iterator();
             while (resources.hasNext()) {
@@ -415,7 +414,7 @@
   }
 
  public static class AnalyzerType extends EnumeratedAttribute {
-    private static Map analyzerLookup = new HashMap();
+    private static Map<String,String> analyzerLookup = new HashMap<String,String>();
 
     static {
       analyzerLookup.put("simple", SimpleAnalyzer.class.getName());
@@ -429,12 +428,12 @@
      */
     @Override
     public String[] getValues() {
-      Set keys = analyzerLookup.keySet();
-      return (String[]) keys.toArray(new String[0]);
+      Set<String> keys = analyzerLookup.keySet();
+      return keys.toArray(new String[0]);
     }
 
     public String getClassname() {
-      return (String) analyzerLookup.get(getValue());
+      return analyzerLookup.get(getValue());
     }
   }
 }

Modified: lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/PerfRunData.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/PerfRunData.java?rev=880963&r1=880962&r2=880963&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/PerfRunData.java (original)
+++ lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/PerfRunData.java Mon Nov 16 21:24:41 2009
@@ -173,25 +173,41 @@
   }
 
   /**
-   * @return Returns the indexReader.
+   * @return Returns the indexReader.  NOTE: this returns a
+   * reference.  You must call IndexReader.decRef() when
+   * you're done.
    */
-  public IndexReader getIndexReader() {
+  public synchronized IndexReader getIndexReader() {
+    if (indexReader != null) {
+      indexReader.incRef();
+    }
     return indexReader;
   }
 
   /**
-   * @return Returns the indexSearcher.
+   * @return Returns the indexSearcher.  NOTE: this returns
+   * a reference to the underlying IndexReader.  You must
+   * call IndexReader.decRef() when you're done.
    */
-  public IndexSearcher getIndexSearcher() {
+  public synchronized IndexSearcher getIndexSearcher() {
+    if (indexReader != null) {
+      indexReader.incRef();
+    }
     return indexSearcher;
   }
 
   /**
    * @param indexReader The indexReader to set.
    */
-  public void setIndexReader(IndexReader indexReader) {
+  public synchronized void setIndexReader(IndexReader indexReader) throws IOException {
+    if (this.indexReader != null) {
+      // Release current IR
+      this.indexReader.decRef();
+    }
     this.indexReader = indexReader;
     if (indexReader != null) {
+      // Hold reference to new IR
+      indexReader.incRef();
       indexSearcher = new IndexSearcher(indexReader);
     } else {
       indexSearcher = null;

Modified: lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/EnwikiContentSource.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/EnwikiContentSource.java?rev=880963&r1=880962&r2=880963&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/EnwikiContentSource.java (original)
+++ lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/EnwikiContentSource.java Mon Nov 16 21:24:41 2009
@@ -24,6 +24,7 @@
 import java.util.Map;
 
 import org.apache.lucene.benchmark.byTask.utils.Config;
+import org.apache.lucene.util.ThreadInterruptedException;
 import org.xml.sax.Attributes;
 import org.xml.sax.InputSource;
 import org.xml.sax.SAXException;
@@ -66,6 +67,7 @@
           try {
             wait();
           } catch (InterruptedException ie) {
+            throw new ThreadInterruptedException(ie);
           }
         }
         if (nmde != null) {
@@ -127,6 +129,7 @@
                 try {
                   wait();
                 } catch (InterruptedException ie) {
+                  throw new ThreadInterruptedException(ie);
                 }
               }
               tuple = tmpTuple;

Modified: lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecContentSource.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecContentSource.java?rev=880963&r1=880962&r2=880963&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecContentSource.java (original)
+++ lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecContentSource.java Mon Nov 16 21:24:41 2009
@@ -33,6 +33,7 @@
 
 import org.apache.lucene.benchmark.byTask.utils.Config;
 import org.apache.lucene.benchmark.byTask.utils.StringBufferReader;
+import org.apache.lucene.util.ThreadInterruptedException;
 
 /**
  * Implements a {@link ContentSource} over the TREC collection.
@@ -48,6 +49,7 @@
  * <li><b>html.parser</b> - specifies the {@link HTMLParser} class to use for
  * parsing the TREC documents content (<b>default=DemoHTMLParser</b>).
  * <li><b>content.source.encoding</b> - if not specified, ISO-8859-1 is used.
+ * <li><b>content.source.excludeIteration</b> - if true, do not append iteration number to docname
  * </ul>
  */
 public class TrecContentSource extends ContentSource {
@@ -91,6 +93,7 @@
   BufferedReader reader;
   int iteration = 0;
   HTMLParser htmlParser;
+  private boolean excludeDocnameIteration;
   
   private DateFormatInfo getDateFormatInfo() {
     DateFormatInfo dfi = dateFormats.get();
@@ -256,7 +259,8 @@
       read(docBuf, DOCNO, true, false, null);
       name = docBuf.substring(DOCNO.length(), docBuf.indexOf(TERMINATING_DOCNO,
           DOCNO.length()));
-      name = name + "_" + iteration;
+      if (!excludeDocnameIteration)
+        name = name + "_" + iteration;
 
       // 3. skip until doc header
       docBuf.setLength(0);
@@ -299,10 +303,8 @@
     try {
       docData = htmlParser.parse(docData, name, date, r, null);
       addDoc();
-    } catch (InterruptedException e) {
-      IOException ex = new IOException(e.getMessage());
-      ex.initCause(e);
-      throw ex;
+    } catch (InterruptedException ie) {
+      throw new ThreadInterruptedException(ie);
     }
 
     return docData;
@@ -342,6 +344,7 @@
     if (encoding == null) {
       encoding = "ISO-8859-1";
     }
+    excludeDocnameIteration = config.get("content.source.excludeIteration", false);
   }
 
 }

Modified: lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/CloseReaderTask.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/CloseReaderTask.java?rev=880963&r1=880962&r2=880963&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/CloseReaderTask.java (original)
+++ lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/CloseReaderTask.java Mon Nov 16 21:24:41 2009
@@ -35,11 +35,12 @@
 
   @Override
   public int doLogic() throws IOException {
-    IndexReader reader= getRunData().getIndexReader();
-    if (reader!=null) {
-      reader.close();
-    }
+    IndexReader reader = getRunData().getIndexReader();
     getRunData().setIndexReader(null);
+    if (reader.getRefCount() != 1) {
+      System.out.println("WARNING: CloseReader: reference count is currently " + reader.getRefCount());
+    }
+    reader.decRef();
     return 1;
   }
 

Modified: lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/CommitIndexTask.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/CommitIndexTask.java?rev=880963&r1=880962&r2=880963&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/CommitIndexTask.java (original)
+++ lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/CommitIndexTask.java Mon Nov 16 21:24:41 2009
@@ -54,6 +54,7 @@
       IndexReader r = getRunData().getIndexReader();
       if (r != null) {
         r.commit(commitUserData);
+        r.decRef();
       } else {
         throw new IllegalStateException("neither IndexWriter nor IndexReader is currently open");
       }

Modified: lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/CreateIndexTask.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/CreateIndexTask.java?rev=880963&r1=880962&r2=880963&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/CreateIndexTask.java (original)
+++ lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/CreateIndexTask.java Mon Nov 16 21:24:41 2009
@@ -127,11 +127,10 @@
     PerfRunData runData = getRunData();
     Config config = runData.getConfig();
     
-    IndexDeletionPolicy indexDeletionPolicy = getIndexDeletionPolicy(config);
-    
     IndexWriter writer = new IndexWriter(runData.getDirectory(),
                                          runData.getAnalyzer(),
-                                         true, indexDeletionPolicy,
+                                         true,
+                                         getIndexDeletionPolicy(config),
                                          IndexWriter.MaxFieldLength.LIMITED);
     setIndexWriterConfig(writer, config);
     runData.setIndexWriter(writer);

Modified: lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/DeleteByPercentTask.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/DeleteByPercentTask.java?rev=880963&r1=880962&r2=880963&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/DeleteByPercentTask.java (original)
+++ lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/DeleteByPercentTask.java Mon Nov 16 21:24:41 2009
@@ -88,6 +88,7 @@
       termDocs.close();
     }
     System.out.println("--> processed (delete) " + numDeleted + " docs");
+    r.decRef();
     return numDeleted;
   }
 }

Modified: lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/DeleteDocTask.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/DeleteDocTask.java?rev=880963&r1=880962&r2=880963&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/DeleteDocTask.java (original)
+++ lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/DeleteDocTask.java Mon Nov 16 21:24:41 2009
@@ -18,6 +18,7 @@
  */
 
 import org.apache.lucene.benchmark.byTask.PerfRunData;
+import org.apache.lucene.index.IndexReader;
 
 /**
  * Delete a document by docid. If no docid param is supplied, deletes doc with
@@ -42,8 +43,10 @@
   
   @Override
   public int doLogic() throws Exception {
-    getRunData().getIndexReader().deleteDocument(docid);
+    IndexReader r = getRunData().getIndexReader();
+    r.deleteDocument(docid);
     lastDeleted = docid;
+    r.decRef();
     return 1; // one work item done here
   }
 

Modified: lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/FlushReaderTask.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/FlushReaderTask.java?rev=880963&r1=880962&r2=880963&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/FlushReaderTask.java (original)
+++ lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/FlushReaderTask.java Mon Nov 16 21:24:41 2009
@@ -52,6 +52,7 @@
     } else {
       reader.flush();
     }
+    reader.decRef();
     return 1;
   }
 }

Modified: lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/NearRealtimeReaderTask.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/NearRealtimeReaderTask.java?rev=880963&r1=880962&r2=880963&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/NearRealtimeReaderTask.java (original)
+++ lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/NearRealtimeReaderTask.java Mon Nov 16 21:24:41 2009
@@ -17,18 +17,9 @@
  * limitations under the License.
  */
 
-import java.io.IOException;
-
 import org.apache.lucene.benchmark.byTask.PerfRunData;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.TermQuery;
-import org.apache.lucene.search.Sort;
-import org.apache.lucene.search.SortField;
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.TopFieldDocs;
-import org.apache.lucene.index.Term;
 
 /**
  * Spawns a BG thread that periodically (defaults to 3.0
@@ -43,97 +34,67 @@
  */
 public class NearRealtimeReaderTask extends PerfTask {
 
-  ReopenThread t;
-  float pauseSec = 3.0f;
+  long pauseMSec = 3000L;
 
-  private static class ReopenThread extends Thread {
+  public NearRealtimeReaderTask(PerfRunData runData) {
+    super(runData);
+  }
 
-    final IndexWriter writer;
-    final int pauseMsec;
+  @Override
+  public int doLogic() throws Exception {
 
-    public volatile boolean done;
+    final PerfRunData runData = getRunData();
 
-    ReopenThread(IndexWriter writer, float pauseSec) {
-      this.writer = writer;
-      this.pauseMsec = (int) (1000*pauseSec);
-      setDaemon(true);
+    // Get initial reader
+    IndexWriter w = runData.getIndexWriter();
+    if (w == null) {
+      throw new RuntimeException("please open the writer before invoking NearRealtimeReader");
     }
 
-    @Override
-    public void run() {
-
-      IndexReader reader = null;
+    if (runData.getIndexReader() != null) {
+      throw new RuntimeException("please close the existing reader before invoking NearRealtimeReader");
+    }
+    
+    long t = System.currentTimeMillis();
+    IndexReader r = w.getReader();
+    runData.setIndexReader(r);
+    // Transfer our reference to runData
+    r.decRef();
+
+    // TODO: gather basic metrics for reporting -- eg mean,
+    // stddev, min/max reopen latencies
+
+    // Parent sequence sets stopNow
+    int reopenCount = 0;
+    while(!stopNow) {
+      long waitForMsec = (long) (pauseMSec - (System.currentTimeMillis() - t));
+      if (waitForMsec > 0) {
+        Thread.sleep(waitForMsec);
+      }
 
-      final Query query = new TermQuery(new Term("body", "1"));
-      final SortField sf = new SortField("docdate", SortField.LONG);
-      final Sort sort = new Sort(sf);
-
-      try {
-        while(!done) {
-          final long t0 = System.currentTimeMillis();
-          if (reader == null) {
-            reader = writer.getReader();
-          } else {
-            final IndexReader newReader = reader.reopen();
-            if (reader != newReader) {
-              reader.close();
-              reader = newReader;
-            }
-          }
-
-          final long t1 = System.currentTimeMillis();
-          final TopFieldDocs hits = new IndexSearcher(reader).search(query, null, 10, sort);
-          final long t2 = System.currentTimeMillis();
-          System.out.println("nrt: open " + (t1-t0) + " msec; search " + (t2-t1) + " msec, " + hits.totalHits +
-                             " results; " + reader.numDocs() + " docs");
-
-          final long t4 = System.currentTimeMillis();
-          final int delay = (int) (pauseMsec - (t4-t0));
-          if (delay > 0) {
-            try {
-              Thread.sleep(delay);
-            } catch (InterruptedException ie) {
-              throw new RuntimeException(ie);
-            }
-          }
-        }
-      } catch (Exception e) {
-        throw new RuntimeException(e);
+      t = System.currentTimeMillis();
+      final IndexReader newReader = r.reopen();
+      if (r != newReader) {
+        // TODO: somehow we need to enable warming, here
+        runData.setIndexReader(newReader);
+        // Transfer our reference to runData
+        newReader.decRef();
+        r = newReader;
+        reopenCount++;
       }
     }
-  }
 
-  public NearRealtimeReaderTask(PerfRunData runData) {
-    super(runData);
-  }
-
-  @Override
-  public int doLogic() throws IOException {
-    if (t == null) {
-      IndexWriter w = getRunData().getIndexWriter();
-      t = new ReopenThread(w, pauseSec);
-      t.start();
-    }
-    return 1;
+    return reopenCount;
   }
 
   @Override
   public void setParams(String params) {
     super.setParams(params);
-    pauseSec = Float.parseFloat(params);
+    pauseMSec = (long) (1000.0*Float.parseFloat(params));
   }
 
   @Override
   public boolean supportsParams() {
     return true;
   }
-
-  // Close the thread
-  @Override
-  public void close() throws InterruptedException {
-    if (t != null) {
-      t.done = true;
-      t.join();
-    }
-  }
 }

Modified: lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/OpenIndexTask.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/OpenIndexTask.java?rev=880963&r1=880962&r2=880963&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/OpenIndexTask.java (original)
+++ lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/OpenIndexTask.java Mon Nov 16 21:24:41 2009
@@ -20,6 +20,7 @@
 import org.apache.lucene.benchmark.byTask.PerfRunData;
 import org.apache.lucene.benchmark.byTask.utils.Config;
 import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexCommit;
 import org.apache.lucene.index.LogMergePolicy;
 
 import java.io.IOException;
@@ -30,6 +31,11 @@
  * <br>Other side effects: index writer object in perfRunData is set.
  * <br>Relevant properties: <code>merge.factor, max.buffered,
  * max.field.length, ram.flush.mb [default 0]</code>.
+ *
+ * <p> Accepts a param specifying the commit point as
+ * previously saved with CommitIndexTask.  If you specify
+ * this, it rolls the index back to that commit on opening
+ * the IndexWriter.
  */
 public class OpenIndexTask extends PerfTask {
 
@@ -37,6 +43,7 @@
   public static final int DEFAULT_MAX_FIELD_LENGTH = IndexWriter.DEFAULT_MAX_FIELD_LENGTH;
   public static final int DEFAULT_MERGE_PFACTOR = LogMergePolicy.DEFAULT_MERGE_FACTOR;
   public static final double DEFAULT_RAM_FLUSH_MB = (int) IndexWriter.DEFAULT_RAM_BUFFER_SIZE_MB;
+  private String commitUserData;
 
   public OpenIndexTask(PerfRunData runData) {
     super(runData);
@@ -46,12 +53,34 @@
   public int doLogic() throws IOException {
     PerfRunData runData = getRunData();
     Config config = runData.getConfig();
+    final IndexCommit ic;
+    if (commitUserData != null) {
+      ic = OpenReaderTask.findIndexCommit(runData.getDirectory(), commitUserData);
+    } else {
+      ic = null;
+    }
+    
     IndexWriter writer = new IndexWriter(runData.getDirectory(),
                                          runData.getAnalyzer(),
-                                         false,
-                                         IndexWriter.MaxFieldLength.UNLIMITED);
+                                         CreateIndexTask.getIndexDeletionPolicy(config),
+                                         IndexWriter.MaxFieldLength.UNLIMITED,
+                                         ic);
     CreateIndexTask.setIndexWriterConfig(writer, config);
     runData.setIndexWriter(writer);
     return 1;
   }
+
+  @Override
+  public void setParams(String params) {
+    super.setParams(params);
+    if (params != null) {
+      // specifies which commit point to open
+      commitUserData = params;
+    }
+  }
+
+  @Override
+  public boolean supportsParams() {
+    return true;
+  }
 }

Modified: lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/OpenReaderTask.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/OpenReaderTask.java?rev=880963&r1=880962&r2=880963&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/OpenReaderTask.java (original)
+++ lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/OpenReaderTask.java Mon Nov 16 21:24:41 2009
@@ -47,35 +47,27 @@
     Directory dir = getRunData().getDirectory();
     Config config = getRunData().getConfig();
     IndexReader r = null;
+    final IndexDeletionPolicy deletionPolicy;
+    if (readOnly) {
+      deletionPolicy = null;
+    } else {
+      deletionPolicy = CreateIndexTask.getIndexDeletionPolicy(config);
+    }
     if (commitUserData != null) {
-      r = openCommitPoint(commitUserData, dir, config, readOnly);
+      r = IndexReader.open(OpenReaderTask.findIndexCommit(dir, commitUserData),
+                           deletionPolicy,
+                           readOnly); 
     } else {
-      IndexDeletionPolicy indexDeletionPolicy = CreateIndexTask.getIndexDeletionPolicy(config);
-      r = IndexReader.open(dir, indexDeletionPolicy, readOnly); 
+      r = IndexReader.open(dir,
+                           deletionPolicy,
+                           readOnly); 
     }
     getRunData().setIndexReader(r);
+    // We transfer reference to the run data
+    r.decRef();
     return 1;
   }
  
-  public static IndexReader openCommitPoint(String userData, Directory dir, Config config, boolean readOnly) throws IOException {
-    IndexReader r = null;
-    Collection<IndexCommit> commits = IndexReader.listCommits(dir);
-    for (final IndexCommit ic : commits) {
-      Map<String,String> map = ic.getUserData();
-      String ud = null;
-      if (map != null) {
-        ud = map.get(USER_DATA);
-      }
-      if (ud != null && ud.equals(userData)) {
-        IndexDeletionPolicy indexDeletionPolicy = CreateIndexTask.getIndexDeletionPolicy(config);
-        r = IndexReader.open(ic, indexDeletionPolicy, readOnly);
-        break;
-      }
-    }
-    if (r == null) throw new IOException("cannot find commitPoint userData:"+userData);
-    return r;
-  }
-  
   @Override
   public void setParams(String params) {
     super.setParams(params);
@@ -94,4 +86,20 @@
   public boolean supportsParams() {
     return true;
   }
+
+  public static IndexCommit findIndexCommit(Directory dir, String userData) throws IOException {
+    Collection<IndexCommit> commits = IndexReader.listCommits(dir);
+    for (final IndexCommit ic : commits) {
+      Map<String,String> map = ic.getUserData();
+      String ud = null;
+      if (map != null) {
+        ud = map.get(USER_DATA);
+      }
+      if (ud != null && ud.equals(userData)) {
+        return ic;
+      }
+    }
+
+    throw new IOException("index does not contain commit with userData: " + userData);
+  }
 }

Modified: lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/PerfTask.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/PerfTask.java?rev=880963&r1=880962&r2=880963&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/PerfTask.java (original)
+++ lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/PerfTask.java Mon Nov 16 21:24:41 2009
@@ -59,7 +59,9 @@
   private int maxDepthLogStart = 0;
   private boolean disableCounting = false;
   protected String params = null;
-  
+
+  private boolean runInBackground;
+
   protected static final String NEW_LINE = System.getProperty("line.separator");
 
   /** Should not be used externally */
@@ -70,25 +72,20 @@
     }
   }
 
-  /**
-   * @deprecated will be removed in 3.0. checks if there are any obsolete
-   *             settings, like doc.add.log.step and doc.delete.log.step and
-   *             alerts the user.
-   */
-  private void checkObsoleteSettings(Config config) {
-    if (config.get("doc.add.log.step", null) != null) {
-      throw new RuntimeException("doc.add.log.step is not supported anymore. " +
-      		"Use log.step.AddDoc and refer to CHANGES to read on the recent " +
-      		"API changes done to Benchmark's DocMaker and Task-based logging.");
-    }
-    
-    if (config.get("doc.delete.log.step", null) != null) {
-      throw new RuntimeException("doc.delete.log.step is not supported anymore. " +
-          "Use log.step.DeleteDoc and refer to CHANGES to read on the recent " +
-          "API changes done to Benchmark's DocMaker and Task-based logging.");
-    }
+  public void setRunInBackground() {
+    runInBackground = true;
   }
-  
+
+  public boolean getRunInBackground() {
+    return runInBackground;
+  }
+
+  protected volatile boolean stopNow;
+
+  public void stopNow() {
+    stopNow = true;
+  }
+
   public PerfTask(PerfRunData runData) {
     this();
     this.runData = runData;
@@ -114,7 +111,6 @@
     if (logStep <= 0) {
       logStep = Integer.MAX_VALUE;
     }
-    checkObsoleteSettings(config);
   }
   
   @Override
@@ -132,9 +128,7 @@
    * @return number of work items done by this task.
    */
   public final int runAndMaybeStats(boolean reportStats) throws Exception {
-    if (reportStats && depth <= maxDepthLogStart && !shouldNeverLogAtStart()) {
-      System.out.println("------------> starting task: " + getName());
-    }
+    stopNow = false;
     if (!reportStats || shouldNotRecordStats()) {
       setup();
       int count = doLogic();
@@ -142,9 +136,12 @@
       tearDown();
       return count;
     }
+    if (reportStats && depth <= maxDepthLogStart && !shouldNeverLogAtStart()) {
+      System.out.println("------------> starting task: " + getName());
+    }
     setup();
     Points pnts = runData.getPoints();
-    TaskStats ts = pnts.markTaskStart(this,runData.getConfig().getRoundNumber());
+    TaskStats ts = pnts.markTaskStart(this, runData.getConfig().getRoundNumber());
     int count = doLogic();
     count = disableCounting ? 0 : count;
     pnts.markTaskEnd(ts, count);
@@ -217,6 +214,9 @@
       sb.append('-');
     }
     sb.append(getName());
+    if (getRunInBackground()) {
+      sb.append(" &");
+    }
     return sb.toString();
   }
 

Modified: lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/PrintReaderTask.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/PrintReaderTask.java?rev=880963&r1=880962&r2=880963&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/PrintReaderTask.java (original)
+++ lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/PrintReaderTask.java Mon Nov 16 21:24:41 2009
@@ -18,7 +18,6 @@
  */
 
 import org.apache.lucene.benchmark.byTask.PerfRunData;
-import org.apache.lucene.benchmark.byTask.utils.Config;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.store.Directory;
 
@@ -43,12 +42,13 @@
   @Override
   public int doLogic() throws Exception {
     Directory dir = getRunData().getDirectory();
-    Config config = getRunData().getConfig();
     IndexReader r = null;
     if (userData == null) 
       r = IndexReader.open(dir, true);
     else
-      r = OpenReaderTask.openCommitPoint(userData, dir, config, true);
+      r = IndexReader.open(OpenReaderTask.findIndexCommit(dir, userData),
+                           null,
+                           true);
     System.out.println("--> numDocs:"+r.numDocs()+" dels:"+r.numDeletedDocs());
     r.close();
     return 1;

Modified: lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ReadTask.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ReadTask.java?rev=880963&r1=880962&r2=880963&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ReadTask.java (original)
+++ lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ReadTask.java Mon Nov 16 21:24:41 2009
@@ -25,7 +25,6 @@
 import java.util.Set;
 
 import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.benchmark.byTask.PerfRunData;
 import org.apache.lucene.benchmark.byTask.feeds.QueryMaker;
 import org.apache.lucene.document.Document;
@@ -35,11 +34,10 @@
 import org.apache.lucene.search.MultiTermQuery;
 import org.apache.lucene.search.TopFieldCollector;
 import org.apache.lucene.search.ScoreDoc;
+import org.apache.lucene.search.Weight;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.Sort;
-import org.apache.lucene.search.highlight.Highlighter;
-import org.apache.lucene.search.highlight.InvalidTokenOffsetsException;
 import org.apache.lucene.store.Directory;
 
 
@@ -60,29 +58,44 @@
  */
 public abstract class ReadTask extends PerfTask {
 
+  private final QueryMaker queryMaker;
+
   public ReadTask(PerfRunData runData) {
     super(runData);
+    if (withSearch()) {
+      queryMaker = getQueryMaker();
+    } else {
+      queryMaker = null;
+    }
   }
   @Override
   public int doLogic() throws Exception {
     int res = 0;
-    boolean closeReader = false;
 
     // open reader or use existing one
-    IndexReader ir = getRunData().getIndexReader();
-    if (ir == null) {
+    IndexSearcher searcher = getRunData().getIndexSearcher();
+
+    IndexReader reader;
+
+    final boolean closeSearcher;
+    if (searcher == null) {
+      // open our own reader
       Directory dir = getRunData().getDirectory();
-      ir = IndexReader.open(dir, true);
-      closeReader = true;
-      //res++; //this is confusing, comment it out
+      reader = IndexReader.open(dir, true);
+      searcher = new IndexSearcher(reader);
+      closeSearcher = true;
+    } else {
+      // use existing one; this passes +1 ref to us
+      reader = searcher.getIndexReader();
+      closeSearcher = false;
     }
 
     // optionally warm and add num docs traversed to count
     if (withWarm()) {
       Document doc = null;
-      for (int m = 0; m < ir.maxDoc(); m++) {
-        if (!ir.isDeleted(m)) {
-          doc = ir.document(m);
+      for (int m = 0; m < reader.maxDoc(); m++) {
+        if (!reader.isDeleted(m)) {
+          doc = reader.document(m);
           res += (doc == null ? 0 : 1);
         }
       }
@@ -90,24 +103,18 @@
 
     if (withSearch()) {
       res++;
-      final IndexSearcher searcher;
-      if (closeReader) {
-        searcher = new IndexSearcher(ir);
-      } else {
-        searcher = getRunData().getIndexSearcher();
-      }
-      QueryMaker queryMaker = getQueryMaker();
       Query q = queryMaker.makeQuery();
       Sort sort = getSort();
       TopDocs hits;
       final int numHits = numHits();
       if (numHits > 0) {
         if (sort != null) {
-          // TODO: change the following to create TFC with in/out-of order
-          // according to whether the query's Scorer.
+          Weight w = q.weight(searcher);
           TopFieldCollector collector = TopFieldCollector.create(sort, numHits,
-              true, withScore(), withMaxScore(), false);
-          searcher.search(q, collector);
+                                                                 true, withScore(),
+                                                                 withMaxScore(),
+                                                                 !w.scoresDocsOutOfOrder());
+          searcher.search(w, null, collector);
           hits = collector.topDocs();
         } else {
           hits = searcher.search(q, numHits);
@@ -115,22 +122,19 @@
 
         final String printHitsField = getRunData().getConfig().get("print.hits.field", null);
         if (printHitsField != null && printHitsField.length() > 0) {
-          final IndexReader r = searcher.getIndexReader();
           if (q instanceof MultiTermQuery) {
             System.out.println("MultiTermQuery term count = " + ((MultiTermQuery) q).getTotalNumberOfTerms());
           }
           System.out.println("totalHits = " + hits.totalHits);
-          System.out.println("maxDoc()  = " + r.maxDoc());
-          System.out.println("numDocs() = " + r.numDocs());
+          System.out.println("maxDoc()  = " + reader.maxDoc());
+          System.out.println("numDocs() = " + reader.numDocs());
           for(int i=0;i<hits.scoreDocs.length;i++) {
             final int docID = hits.scoreDocs[i].doc;
-            final Document doc = r.document(docID);
+            final Document doc = reader.document(docID);
             System.out.println("  " + i + ": doc=" + docID + " score=" + hits.scoreDocs[i].score + " " + printHitsField + " =" + doc.get(printHitsField));
           }
         }
 
-        //System.out.println("q=" + q + ":" + hits.totalHits + " total hits"); 
-
         if (withTraverse()) {
           final ScoreDoc[] scoreDocs = hits.scoreDocs;
           int traversalSize = Math.min(scoreDocs.length, traversalSize());
@@ -147,13 +151,13 @@
               int id = scoreDocs[m].doc;
               res++;
               if (retrieve) {
-                Document document = retrieveDoc(ir, id);
+                Document document = retrieveDoc(reader, id);
                 res += document != null ? 1 : 0;
                 if (numHighlight > 0 && m < numHighlight) {
                   Collection<String> fieldsToHighlight = getFieldsToHighlight(document);
                   for (final String field : fieldsToHighlight) {
                     String text = document.get(field);
-                    res += highlighter.doHighlight(ir, id, field, document, analyzer, text);
+                    res += highlighter.doHighlight(reader, id, field, document, analyzer, text);
                   }
                 }
               }
@@ -161,12 +165,13 @@
           }
         }
       }
-
-      searcher.close();
     }
 
-    if (closeReader) {
-      ir.close();
+    if (closeSearcher) {
+      searcher.close();
+    } else {
+      // Release our +1 ref from above
+      reader.decRef();
     }
     return res;
   }
@@ -256,53 +261,12 @@
   }
 
   /**
-   * @deprecated Use {@link #getBenchmarkHighlighter(Query)}
-   */
-  final Highlighter getHighlighter(Query q) {
-    // not called
-    return null;
-  }
-  
-  /**
    * Return an appropriate highlighter to be used with
    * highlighting tasks
    */
   protected BenchmarkHighlighter getBenchmarkHighlighter(Query q){
     return null;
   }
-
-  /**
-   * @return the maximum number of highlighter fragments
-   * @deprecated Please define getBenchmarkHighlighter instead
-   */
-  final int maxNumFragments(){
-    // not called -- we switched this method to final to
-    // force any external subclasses to cutover to
-    // getBenchmarkHighlighter instead
-    return 10;
-  }
-
-  /**
-   *
-   * @return true if the highlighter should merge contiguous fragments
-   * @deprecated Please define getBenchmarkHighlighter instead
-   */
-  final boolean isMergeContiguousFragments(){
-    // not called -- we switched this method to final to
-    // force any external subclasses to cutover to
-    // getBenchmarkHighlighter instead
-    return false;
-  }
-
-  /**
-   * @deprecated Please define getBenchmarkHighlighter instead
-   */
-  final int doHighlight(TokenStream ts, String text,  Highlighter highlighter, boolean mergeContiguous, int maxFragments) throws IOException, InvalidTokenOffsetsException {
-    // not called -- we switched this method to final to
-    // force any external subclasses to cutover to
-    // getBenchmarkHighlighter instead
-    return 0;
-  }
   
   protected Sort getSort() {
     return null;

Modified: lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ReopenReaderTask.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ReopenReaderTask.java?rev=880963&r1=880962&r2=880963&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ReopenReaderTask.java (original)
+++ lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ReopenReaderTask.java Mon Nov 16 21:24:41 2009
@@ -33,13 +33,13 @@
 
   @Override
   public int doLogic() throws IOException {
-    IndexReader ir = getRunData().getIndexReader();
-    IndexReader or = ir;
-    IndexReader nr = ir.reopen();
-    if(nr != or) {
+    IndexReader r = getRunData().getIndexReader();
+    IndexReader nr = r.reopen();
+    if (nr != r) {
       getRunData().setIndexReader(nr);
-      or.close();
+      nr.decRef();
     }
+    r.decRef();
     return 1;
   }
 }

Added: lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/RollbackIndexTask.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/RollbackIndexTask.java?rev=880963&view=auto
==============================================================================
--- lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/RollbackIndexTask.java (added)
+++ lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/RollbackIndexTask.java Mon Nov 16 21:24:41 2009
@@ -0,0 +1,52 @@
+package org.apache.lucene.benchmark.byTask.tasks;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.io.PrintStream;
+
+import org.apache.lucene.benchmark.byTask.PerfRunData;
+import org.apache.lucene.index.IndexWriter;
+
+/**
+ * Rollback the index writer.
+ */
+public class RollbackIndexTask extends PerfTask {
+
+  public RollbackIndexTask(PerfRunData runData) {
+    super(runData);
+  }
+
+  boolean doWait = true;
+
+  @Override
+  public int doLogic() throws IOException {
+    IndexWriter iw = getRunData().getIndexWriter();
+    if (iw != null) {
+      // If infoStream was set to output to a file, close it.
+      PrintStream infoStream = iw.getInfoStream();
+      if (infoStream != null && infoStream != System.out
+          && infoStream != System.err) {
+        infoStream.close();
+      }
+      iw.rollback();
+      getRunData().setIndexWriter(null);
+    }
+    return 1;
+  }
+}

Modified: lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/TaskSequence.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/TaskSequence.java?rev=880963&r1=880962&r2=880963&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/TaskSequence.java (original)
+++ lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/TaskSequence.java Mon Nov 16 21:24:41 2009
@@ -18,6 +18,7 @@
  */
 
 import java.util.ArrayList;
+import java.util.List;
 import java.text.NumberFormat;
 
 import org.apache.lucene.benchmark.byTask.PerfRunData;
@@ -131,6 +132,33 @@
     return ( parallel ? doParallelTasks() : doSerialTasks());
   }
 
+  private static class RunBackgroundTask extends Thread {
+    private final PerfTask task;
+    private final boolean letChildReport;
+    private volatile int count;
+
+    public RunBackgroundTask(PerfTask task, boolean letChildReport) {
+      this.task = task;
+      this.letChildReport = letChildReport;
+    }
+
+    public void stopNow() throws InterruptedException {
+      task.stopNow();
+    }
+
+    public int getCount() {
+      return count;
+    }
+
+    public void run() {
+      try {
+        count = task.runAndMaybeStats(letChildReport);
+      } catch (Exception e) {
+        throw new RuntimeException(e);
+      }
+    }
+  }
+
   private int doSerialTasks() throws Exception {
     if (rate > 0) {
       return doSerialTasksWithRate();
@@ -142,22 +170,46 @@
     final long t0 = System.currentTimeMillis();
 
     final long runTime = (long) (runTimeSec*1000);
+    List<RunBackgroundTask> bgTasks = null;
 
     for (int k=0; fixedTime || (repetitions==REPEAT_EXHAUST && !exhausted) || k<repetitions; k++) {
-      for(int l=0;l<tasksArray.length;l++)
-        try {
-          final PerfTask task = tasksArray[l];
-          count += task.runAndMaybeStats(letChildReport);
-          if (anyExhaustibleTasks)
-            updateExhausted(task);
-        } catch (NoMoreDataException e) {
-          exhausted = true;
+      if (stopNow) {
+        break;
+      }
+      for(int l=0;l<tasksArray.length;l++) {
+        final PerfTask task = tasksArray[l];
+        if (task.getRunInBackground()) {
+          if (bgTasks == null) {
+            bgTasks = new ArrayList<RunBackgroundTask>();
+          }
+          RunBackgroundTask bgTask = new RunBackgroundTask(task, letChildReport);
+          bgTask.start();
+          bgTasks.add(bgTask);
+        } else {
+          try {
+            count += task.runAndMaybeStats(letChildReport);
+            if (anyExhaustibleTasks)
+              updateExhausted(task);
+          } catch (NoMoreDataException e) {
+            exhausted = true;
+          }
         }
+      }
       if (fixedTime && System.currentTimeMillis()-t0 > runTime) {
         repetitions = k+1;
         break;
       }
     }
+
+    if (bgTasks != null) {
+      for(RunBackgroundTask bgTask : bgTasks) {
+        bgTask.stopNow();
+      }
+      for(RunBackgroundTask bgTask : bgTasks) {
+        bgTask.join();
+        count += bgTask.getCount();
+      }
+    }
     return count;
   }
 
@@ -167,12 +219,22 @@
     long nextStartTime = System.currentTimeMillis();
     int count = 0;
     for (int k=0; (repetitions==REPEAT_EXHAUST && !exhausted) || k<repetitions; k++) {
+      if (stopNow) {
+        break;
+      }
       for (int l=0;l<tasksArray.length;l++) {
         final PerfTask task = tasksArray[l];
-        long waitMore = nextStartTime - System.currentTimeMillis();
-        if (waitMore > 0) {
-          //System.out.println("wait: "+waitMore+" for rate: "+ratePerMin+" (delayStep="+delayStep+")");
-          Thread.sleep(waitMore);
+        while(!stopNow) {
+          long waitMore = nextStartTime - System.currentTimeMillis();
+          if (waitMore > 0) {
+            // TODO: better to use condition to notify
+            Thread.sleep(1);
+          } else {
+            break;
+          }
+        }
+        if (stopNow) {
+          break;
         }
         nextStartTime += delayStep; // this aims at avarage rate. 
         try {
@@ -204,46 +266,71 @@
     }
   }
 
+  private class ParallelTask extends Thread {
+
+    public int count;
+    public final PerfTask task;
+
+    public ParallelTask(PerfTask task) {
+      this.task = task;
+    }
+
+    @Override
+    public void run() {
+      try {
+        int n = task.runAndMaybeStats(letChildReport);
+        if (anyExhaustibleTasks) {
+          updateExhausted(task);
+        }
+        count += n;
+      } catch (NoMoreDataException e) {
+        exhausted = true;
+      } catch (Exception e) {
+        throw new RuntimeException(e);
+      }
+    }
+  }
+
+  @Override
+  public void stopNow() {
+    super.stopNow();
+    // Forwards top request to children
+    if (runningParallelTasks != null) {
+      for(ParallelTask t : runningParallelTasks) {
+        t.task.stopNow();
+      }
+    }
+  }
+
+  ParallelTask[] runningParallelTasks;
+
   private int doParallelTasks() throws Exception {
     initTasksArray();
-    final int count [] = {0};
-    Thread t[] = new Thread [repetitions * tasks.size()];
+    ParallelTask t[] = runningParallelTasks = new ParallelTask[repetitions * tasks.size()];
     // prepare threads
-    int indx = 0;
+    int index = 0;
     for (int k=0; k<repetitions; k++) {
       for (int i = 0; i < tasksArray.length; i++) {
         final PerfTask task = (PerfTask) tasksArray[i].clone();
-        t[indx++] = new Thread() {
-          @Override
-          public void run() {
-            try {
-              int n = task.runAndMaybeStats(letChildReport);
-              if (anyExhaustibleTasks)
-                updateExhausted(task);
-              synchronized (count) {
-                count[0] += n;
-              }
-            } catch (NoMoreDataException e) {
-              exhausted = true;
-            } catch (Exception e) {
-              throw new RuntimeException(e);
-            }
-          }
-        };
+        t[index++] = new ParallelTask(task);
       }
     }
     // run threads
     startThreads(t);
+
     // wait for all threads to complete
+    int count = 0;
     for (int i = 0; i < t.length; i++) {
       t[i].join();
+      count += t[i].count;
     }
+
     // return total count
-    return count[0];
+    return count;
   }
 
   // run threads
-  private void startThreads(Thread[] t) throws InterruptedException {
+  private void startThreads(ParallelTask[] t) throws InterruptedException {
     if (rate > 0) {
       startlThreadsWithRate(t);
       return;
@@ -254,13 +341,12 @@
   }
 
   // run threads with rate
-  private void startlThreadsWithRate(Thread[] t) throws InterruptedException {
+  private void startlThreadsWithRate(ParallelTask[] t) throws InterruptedException {
     long delayStep = (perMin ? 60000 : 1000) /rate;
     long nextStartTime = System.currentTimeMillis();
     for (int i = 0; i < t.length; i++) {
       long waitMore = nextStartTime - System.currentTimeMillis();
       if (waitMore > 0) {
-        //System.out.println("thread wait: "+waitMore+" for rate: "+ratePerMin+" (delayStep="+delayStep+")");
         Thread.sleep(waitMore);
       }
       nextStartTime += delayStep; // this aims at average rate of starting threads. 
@@ -298,6 +384,9 @@
     if (rate>0) {
       sb.append(",  rate: " + rate+"/"+(perMin?"min":"sec"));
     }
+    if (getRunInBackground()) {
+      sb.append(" &");
+    }
     return sb.toString();
   }
 

Modified: lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/UpdateDocTask.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/UpdateDocTask.java?rev=880963&r1=880962&r2=880963&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/UpdateDocTask.java (original)
+++ lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/UpdateDocTask.java Mon Nov 16 21:24:41 2009
@@ -21,6 +21,7 @@
 import org.apache.lucene.benchmark.byTask.feeds.DocMaker;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.index.Term;
+import org.apache.lucene.index.IndexWriter;
 
 /**
  * Update a document, using IndexWriter.updateDocument,
@@ -62,7 +63,8 @@
     if (docID == null) {
       throw new IllegalStateException("document must define the docid field");
     }
-    getRunData().getIndexWriter().updateDocument(new Term(DocMaker.ID_FIELD, docID), doc);
+    final IndexWriter iw = getRunData().getIndexWriter();
+    iw.updateDocument(new Term(DocMaker.ID_FIELD, docID), doc);
     return 1;
   }
 

Added: lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/WaitTask.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/WaitTask.java?rev=880963&view=auto
==============================================================================
--- lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/WaitTask.java (added)
+++ lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/WaitTask.java Mon Nov 16 21:24:41 2009
@@ -0,0 +1,75 @@
+package org.apache.lucene.benchmark.byTask.tasks;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.benchmark.byTask.PerfRunData;
+
+/**
+ * Simply waits for the specified (via the parameter) amount
+ * of time.  For example Wait(30s) waits for 30 seconds.
+ * This is useful with background tasks to control how long
+ * the tasks run.
+ *
+ *<p>You can specify h, m, or s (hours, minutes, seconds) as
+ *the trailing time unit.  No unit is interpreted as
+ *seconds.</p>
+ */
+public class WaitTask extends PerfTask {
+
+  private double waitTimeSec;
+
+  public WaitTask(PerfRunData runData) {
+    super(runData);
+  }
+
+  @Override
+  public void setParams(String params) {
+    super.setParams(params);
+    if (params != null) {
+      int multiplier;
+      if (params.endsWith("s")) {
+        multiplier = 1;
+        params = params.substring(0, params.length()-1);
+      } else if (params.endsWith("m")) {
+        multiplier = 60;
+        params = params.substring(0, params.length()-1);
+      } else if (params.endsWith("h")) {
+        multiplier = 3600;
+        params = params.substring(0, params.length()-1);
+      } else {
+        // Assume seconds
+        multiplier = 1;
+      }
+
+      waitTimeSec = Double.parseDouble(params) * multiplier;
+    } else {
+      throw new IllegalArgumentException("you must specify the wait time, eg: 10.0s, 4.5m, 2h");
+    }
+  }
+
+  @Override
+  public int doLogic() throws Exception {
+    Thread.sleep((long) (1000*waitTimeSec));
+    return 0;
+  }
+
+  @Override
+  public boolean supportsParams() {
+    return true;
+  }
+}

Modified: lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/WriteLineDocTask.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/WriteLineDocTask.java?rev=880963&r1=880962&r2=880963&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/WriteLineDocTask.java (original)
+++ lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/WriteLineDocTask.java Mon Nov 16 21:24:41 2009
@@ -36,7 +36,7 @@
  * A task which writes documents, one line per document. Each line is in the
  * following format: title &lt;TAB&gt; date &lt;TAB&gt; body. The output of this
  * task can be consumed by
- * {@link org.apache.lucene.benchmark.byTask.feeds.LineDocMaker} and is intended
+ * {@link org.apache.lucene.benchmark.byTask.feeds.LineDocSource} and is intended
  * to save the IO overhead of opening a file per document to be indexed.<br>
  * Supports the following parameters:
  * <ul>

Modified: lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/utils/Algorithm.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/utils/Algorithm.java?rev=880963&r1=880962&r2=880963&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/utils/Algorithm.java (original)
+++ lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/utils/Algorithm.java Mon Nov 16 21:24:41 2009
@@ -186,6 +186,19 @@
               currSequence = seq2;
               colonOk = false;
               break;
+
+            case '&' :
+              if (currSequence.isParallel()) {
+                throw new Exception("Can only create background tasks within a serial task");
+              }
+              if (prevTask == null) {
+                throw new Exception("& was unexpected");
+              } else if (prevTask.getRunInBackground()) {
+                throw new Exception("double & was unexpected");
+              } else {
+                prevTask.setRunInBackground();
+              }
+              break;
     
             case '>' :
               currSequence.setNoChildReport();

Modified: lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/utils/Format.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/utils/Format.java?rev=880963&r1=880962&r2=880963&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/utils/Format.java (original)
+++ lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/utils/Format.java Mon Nov 16 21:24:41 2009
@@ -112,7 +112,7 @@
    * @param cls class whose simple name is required 
    * @return simple class name
    */
-  public static String simpleName (Class cls) {
+  public static String simpleName (Class<?> cls) {
     String c = cls.getName();
     String p = cls.getPackage().getName();
     int k = c.lastIndexOf(p+".");

Modified: lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/quality/QualityQuery.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/quality/QualityQuery.java?rev=880963&r1=880962&r2=880963&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/quality/QualityQuery.java (original)
+++ lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/quality/QualityQuery.java Mon Nov 16 21:24:41 2009
@@ -32,7 +32,7 @@
  * or each pair as its own field, etc., - and this of course must match the way the 
  * searched index was constructed.
  */
-public class QualityQuery implements Comparable {
+public class QualityQuery implements Comparable<QualityQuery> {
   private String queryID;
   private Map<String,String> nameValPairs;
 
@@ -71,8 +71,7 @@
 
   /* for a nicer sort of input queries before running them.
    * Try first as ints, fall back to string if not int. */ 
-  public int compareTo(Object o) {
-    QualityQuery other = (QualityQuery) o;
+  public int compareTo(QualityQuery other) {
     try {
       // compare as ints when ids ints
       int n = Integer.parseInt(queryID);

Modified: lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/quality/trec/QueryDriver.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/quality/trec/QueryDriver.java?rev=880963&r1=880962&r2=880963&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/quality/trec/QueryDriver.java (original)
+++ lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/quality/trec/QueryDriver.java Mon Nov 16 21:24:41 2009
@@ -38,10 +38,18 @@
  **/
 public class QueryDriver {
   public static void main(String[] args) throws Exception {
-
+    if (args.length != 4) {
+      System.err.println("Usage: QueryDriver <topicsFile> <qrelsFile> <submissionFile> <indexDir>");
+      System.err.println("topicsFile: input file containing queries");
+      System.err.println("qrelsFile: input file containing relevance judgements");
+      System.err.println("submissionFile: output submission file for trec_eval");
+      System.err.println("indexDir: index directory");
+      System.exit(1);
+    }
     
     File topicsFile = new File(args[0]);
     File qrelsFile = new File(args[1]);
+    SubmissionReport submitLog = new SubmissionReport(new PrintWriter(args[2]), "lucene");
     FSDirectory dir = FSDirectory.open(new File(args[3]));
     Searcher searcher = new IndexSearcher(dir, true);
 
@@ -66,7 +74,6 @@
     // run the benchmark
     QualityBenchmark qrun = new QualityBenchmark(qqs, qqParser, searcher, docNameField);
     qrun.setMaxResults(maxResults);
-    SubmissionReport submitLog = null;
     QualityStats stats[] = qrun.execute(judge, submitLog, logger);
 
     // print an avarage sum of the results

Modified: lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/quality/utils/QualityQueriesFinder.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/quality/utils/QualityQueriesFinder.java?rev=880963&r1=880962&r2=880963&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/quality/utils/QualityQueriesFinder.java (original)
+++ lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/quality/utils/QualityQueriesFinder.java Mon Nov 16 21:24:41 2009
@@ -21,9 +21,7 @@
 
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.Term;
-import org.apache.lucene.index.TermsEnum;
-import org.apache.lucene.index.Terms;
-import org.apache.lucene.index.TermRef;
+import org.apache.lucene.index.TermEnum;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.FSDirectory;
 import org.apache.lucene.util.PriorityQueue;
@@ -90,19 +88,15 @@
     IndexReader ir = IndexReader.open(dir, true);
     try {
       int threshold = ir.maxDoc() / 10; // ignore words too common.
-      Terms terms = ir.fields().terms(field);
-      if (terms != null) {
-        TermsEnum termsEnum = terms.iterator();
-        while(true) {
-          TermRef term = termsEnum.next();
-          if (term == null) {
-            break;
-          }
-          int df = termsEnum.docFreq();
-          if (df<threshold) {
-            String ttxt = term.toString();
-            pq.insertWithOverflow(new TermDf(ttxt,df));
-          }
+      TermEnum terms = ir.terms(new Term(field,""));
+      while (terms.next()) {
+        if (!field.equals(terms.term().field())) {
+          break;
+        }
+        int df = terms.docFreq();
+        if (df<threshold) {
+          String ttxt = terms.term().text();
+          pq.insertWithOverflow(new TermDf(ttxt,df));
         }
       }
     } finally {

Modified: lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/utils/ExtractWikipedia.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/utils/ExtractWikipedia.java?rev=880963&r1=880962&r2=880963&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/utils/ExtractWikipedia.java (original)
+++ lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/utils/ExtractWikipedia.java Mon Nov 16 21:24:41 2009
@@ -23,7 +23,6 @@
 import java.util.Properties;
 
 import org.apache.lucene.benchmark.byTask.feeds.DocMaker;
-import org.apache.lucene.benchmark.byTask.feeds.EnwikiDocMaker;
 import org.apache.lucene.benchmark.byTask.feeds.NoMoreDataException;
 import org.apache.lucene.benchmark.byTask.utils.Config;
 import org.apache.lucene.document.Document;
@@ -125,16 +124,16 @@
       }
 
     }
-    DocMaker docMaker = new EnwikiDocMaker();
+    DocMaker docMaker = new DocMaker();
     Properties properties = new Properties();
-
+    properties.setProperty("content.source", "org.apache.lucene.benchmark.byTask.feeds.EnwikiContentSource");
     properties.setProperty("docs.file", wikipedia.getAbsolutePath());
     properties.setProperty("content.source.forever", "false");
     properties.setProperty("keep.image.only.docs", String.valueOf(keepImageOnlyDocs));
     docMaker.setConfig(new Config(properties));
     docMaker.resetInputs();
     if (wikipedia != null && wikipedia.exists()) {
-      System.out.println("Extracting Wikipedia to: " + outputDir + " using EnwikiDocMaker");
+      System.out.println("Extracting Wikipedia to: " + outputDir + " using EnwikiContentSource");
       outputDir.mkdirs();
       ExtractWikipedia extractor = new ExtractWikipedia(docMaker, outputDir);
       extractor.extract();
@@ -145,9 +144,8 @@
 
   private static void printUsage() {
     System.err.println("Usage: java -cp <...> org.apache.lucene.benchmark.utils.ExtractWikipedia --input|-i <Path to Wikipedia XML file> " +
-            "[--output|-o <Output Path>] [--discardImageOnlyDocs|-d] [--useLineDocMaker|-l]");
+            "[--output|-o <Output Path>] [--discardImageOnlyDocs|-d]");
     System.err.println("--discardImageOnlyDocs tells the extractor to skip Wiki docs that contain only images");
-    System.err.println("--useLineDocMaker uses the LineDocMaker.  Default is EnwikiDocMaker");
   }
 
 }
\ No newline at end of file

Modified: lucene/java/branches/flex_1458/contrib/db/bdb-je/src/java/org/apache/lucene/store/je/JEDirectory.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/db/bdb-je/src/java/org/apache/lucene/store/je/JEDirectory.java?rev=880963&r1=880962&r2=880963&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/db/bdb-je/src/java/org/apache/lucene/store/je/JEDirectory.java (original)
+++ lucene/java/branches/flex_1458/contrib/db/bdb-je/src/java/org/apache/lucene/store/je/JEDirectory.java Mon Nov 16 21:24:41 2009
@@ -51,7 +51,7 @@
 
 public class JEDirectory extends Directory {
 
-    protected Set openFiles = Collections.synchronizedSet(new HashSet());
+    protected Set<JEIndexOutput> openFiles = Collections.synchronizedSet(new HashSet<JEIndexOutput>());
 
     protected Database files, blocks;
 
@@ -102,11 +102,11 @@
      * @see #setTransaction
      */
     public void flush() throws IOException {
-        Iterator iterator = openFiles.iterator();
+        Iterator<JEIndexOutput> iterator = openFiles.iterator();
 
         while (iterator.hasNext()) {
             System.out
-                    .println(((JEIndexOutput) iterator.next()).file.getName());
+                    .println(iterator.next().file.getName());
             // ((IndexOutput) iterator.next()).flush();
         }
     }
@@ -149,7 +149,7 @@
     @Override
     public String[] listAll() throws IOException {
         Cursor cursor = null;
-        List list = new ArrayList();
+        List<String> list = new ArrayList<String>();
 
         try {
             try {
@@ -186,7 +186,7 @@
             throw new IOException(e.getMessage());
         }
 
-        return (String[]) list.toArray(new String[list.size()]);
+        return list.toArray(new String[list.size()]);
     }
 
     @Override



Mime
View raw message