lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mikemcc...@apache.org
Subject svn commit: r1369470 [20/23] - in /lucene/dev/branches/pforcodec_3892: ./ dev-tools/ dev-tools/eclipse/ dev-tools/maven/ dev-tools/scripts/ lucene/ lucene/analysis/ lucene/analysis/common/ lucene/analysis/common/src/java/org/tartarus/snowball/ext/ luce...
Date Sat, 04 Aug 2012 21:26:15 GMT
Modified: lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/snowball/TestSnowball.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/snowball/TestSnowball.java?rev=1369470&r1=1369469&r2=1369470&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/snowball/TestSnowball.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/snowball/TestSnowball.java Sat Aug  4 21:26:10 2012
@@ -25,6 +25,7 @@ import org.apache.lucene.analysis.Analyz
 import org.apache.lucene.analysis.MockTokenizer;
 import org.apache.lucene.analysis.Tokenizer;
 import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Analyzer.TokenStreamComponents;
 import org.apache.lucene.analysis.core.KeywordTokenizer;
 import org.apache.lucene.analysis.standard.StandardAnalyzer;
 import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
@@ -116,4 +117,21 @@ public class TestSnowball extends BaseTo
       checkOneTermReuse(a, "", "");
     }
   }
+  
+  public void testRandomStrings() throws IOException {
+    for (String lang : SNOWBALL_LANGS) {
+      checkRandomStrings(lang);
+    }
+  }
+  
+  public void checkRandomStrings(final String snowballLanguage) throws IOException {
+    Analyzer a = new Analyzer() {
+      @Override
+      protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
+        Tokenizer t = new MockTokenizer(reader);
+        return new TokenStreamComponents(t, new SnowballFilter(t, snowballLanguage));
+      }  
+    };
+    checkRandomData(random(), a, 1000*RANDOM_MULTIPLIER);
+  }
 }
\ No newline at end of file

Modified: lucene/dev/branches/pforcodec_3892/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/TaskSequence.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/TaskSequence.java?rev=1369470&r1=1369469&r2=1369470&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/TaskSequence.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/TaskSequence.java Sat Aug  4 21:26:10 2012
@@ -331,7 +331,9 @@ public class TaskSequence extends PerfTa
     // Forwards top request to children
     if (runningParallelTasks != null) {
       for(ParallelTask t : runningParallelTasks) {
-        t.task.stopNow();
+        if (t != null) {
+          t.task.stopNow();
+        }
       }
     }
   }
@@ -355,6 +357,12 @@ public class TaskSequence extends PerfTa
     // run threads
     startThreads(t);
 
+    if (stopNow) {
+      for (ParallelTask task : t) {
+        task.task.stopNow();
+      }
+    }
+
     // wait for all threads to complete
     int count = 0;
     for (int i = 0; i < t.length; i++) {

Modified: lucene/dev/branches/pforcodec_3892/lucene/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/build.xml?rev=1369470&r1=1369469&r2=1369470&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/build.xml (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/build.xml Sat Aug  4 21:26:10 2012
@@ -35,10 +35,9 @@
                         MIGRATE.txt,JRE_VERSION_MIGRATION.txt,
                         CHANGES.txt,
                         **/lib/*.jar,
-                        **/lib/*LICENSE*.txt,
-                        **/lib/*NOTICE*.txt,
+                        licenses/**,
                         */docs/,**/README*"
-              excludes="build/**,site/**"
+              excludes="build/**,site/**,tools/**"
   />
 
 
@@ -572,4 +571,8 @@
     </sequential>
   </target>
 
+  <target name="jar-checksums" depends="clean-jars,resolve">
+    <jar-checksum-macro srcdir="${common.dir}" dstdir="${common.dir}/licenses"/>
+  </target>
+
 </project>

Modified: lucene/dev/branches/pforcodec_3892/lucene/common-build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/common-build.xml?rev=1369470&r1=1369469&r2=1369470&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/common-build.xml (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/common-build.xml Sat Aug  4 21:26:10 2012
@@ -1567,6 +1567,43 @@ ${tests-output}/junit4-*.suites     - pe
     </sequential>
   </macrodef>
 
+  <macrodef name="jar-checksum-macro">
+      <attribute name="srcdir"/>
+      <attribute name="dstdir"/>
+    <sequential>
+      <delete>
+        <fileset dir="@{dstdir}">
+          <include name="**/*.jar.sha1"/>
+        </fileset>
+      </delete>
+
+      <!-- checksum task does not have a flatten=true -->
+      <tempfile property="jar-checksum.temp.dir"/>
+      <mkdir dir="${jar-checksum.temp.dir}"/>
+      <copy todir="${jar-checksum.temp.dir}" flatten="true">
+        <fileset dir="@{srcdir}">
+          <include name="**/*.jar"/>
+          <!-- todo make this something passed into the macro and not some hardcoded set -->
+          <exclude name="build/**"/>
+          <exclude name="dist/**"/>
+          <exclude name="package/**"/>
+          <exclude name="example/exampledocs/**"/>
+        </fileset>
+      </copy>
+
+      <checksum algorithm="SHA1" fileext=".sha1" todir="@{dstdir}">
+        <fileset dir="${jar-checksum.temp.dir}"/>
+      </checksum>
+
+      <delete dir="${jar-checksum.temp.dir}"/>
+
+      <fixcrlf 
+        srcdir="@{dstdir}"
+        includes="**/*.jar.sha1"
+        eol="lf" fixlast="true" encoding="US-ASCII" />
+    </sequential>
+  </macrodef>
+
   <macrodef name="sign-artifacts-macro">
     <attribute name="artifacts.dir"/>
     <sequential>

Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/PostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/PostingsFormat.java?rev=1369470&r1=1369469&r2=1369470&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/PostingsFormat.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/PostingsFormat.java Sat Aug  4 21:26:10 2012
@@ -53,7 +53,13 @@ public abstract class PostingsFormat imp
 
   /** Reads a segment.  NOTE: by the time this call
    *  returns, it must hold open any files it will need to
-   *  use; else, those files may be deleted. */
+   *  use; else, those files may be deleted. 
+   *  Additionally, required files may be deleted during the execution of 
+   *  this call before there is a chance to open them. Under these 
+   *  circumstances an IOException should be thrown by the implementation. 
+   *  IOExceptions are expected and will automatically cause a retry of the 
+   *  segment opening logic with the newly revised segments.
+   *  */
   public abstract FieldsProducer fieldsProducer(SegmentReadState state) throws IOException;
 
   @Override

Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/document/Field.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/document/Field.java?rev=1369470&r1=1369469&r2=1369470&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/document/Field.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/document/Field.java Sat Aug  4 21:26:10 2012
@@ -377,6 +377,11 @@ public class Field implements IndexableF
    * @see org.apache.lucene.search.similarities.DefaultSimilarity#encodeNormValue(float)
    */
   public void setBoost(float boost) {
+    if (boost != 1.0f) {
+      if (type.indexed() == false || type.omitNorms()) {
+        throw new IllegalArgumentException("You cannot set an index-time boost on an unindexed field, or one that omits norms");
+      }
+    }
     this.boost = boost;
   }
 

Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/document/StoredField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/document/StoredField.java?rev=1369470&r1=1369469&r2=1369470&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/document/StoredField.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/document/StoredField.java Sat Aug  4 21:26:10 2012
@@ -49,6 +49,7 @@ public final class StoredField extends F
     super(name, value, TYPE);
   }
 
+  // TODO: not great but maybe not a big problem?
   public StoredField(String name, int value) {
     super(name, TYPE);
     fieldsData = value;

Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/document/StringField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/document/StringField.java?rev=1369470&r1=1369469&r2=1369470&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/document/StringField.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/document/StringField.java Sat Aug  4 21:26:10 2012
@@ -54,9 +54,4 @@ public final class StringField extends F
   public StringField(String name, String value, Store stored) {
     super(name, value, stored == Store.YES ? TYPE_STORED : TYPE_NOT_STORED);
   }
-
-  @Override
-  public String stringValue() {
-    return (fieldsData == null) ? null : fieldsData.toString();
-  }
 }

Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/document/TextField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/document/TextField.java?rev=1369470&r1=1369469&r2=1369470&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/document/TextField.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/document/TextField.java Sat Aug  4 21:26:10 2012
@@ -46,9 +46,9 @@ public final class TextField extends Fie
 
   // TODO: add sugar for term vectors...?
 
-  /** Creates a new TextField with Reader value. */
-  public TextField(String name, Reader reader, Store store) {
-    super(name, reader, store == Store.YES ? TYPE_STORED : TYPE_NOT_STORED);
+  /** Creates a new un-stored TextField with Reader value. */
+  public TextField(String name, Reader reader) {
+    super(name, reader, TYPE_NOT_STORED);
   }
 
   /** Creates a new TextField with String value. */

Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/AtomicReaderContext.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/AtomicReaderContext.java?rev=1369470&r1=1369469&r2=1369470&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/AtomicReaderContext.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/AtomicReaderContext.java Sat Aug  4 21:26:10 2012
@@ -1,8 +1,5 @@
 package org.apache.lucene.index;
 
-import java.util.Collections;
-import java.util.List;
-
 /*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
@@ -20,6 +17,9 @@ import java.util.List;
  * limitations under the License.
  */
 
+import java.util.Collections;
+import java.util.List;
+
 /**
  * {@link IndexReaderContext} for {@link AtomicReader} instances
  * @lucene.experimental
@@ -51,8 +51,9 @@ public final class AtomicReaderContext e
   
   @Override
   public List<AtomicReaderContext> leaves() {
-    if (!isTopLevel)
+    if (!isTopLevel) {
       throw new UnsupportedOperationException("This is not a top-level context.");
+    }
     assert leaves != null;
     return leaves;
   }

Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/DirectoryReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/DirectoryReader.java?rev=1369470&r1=1369469&r2=1369470&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/DirectoryReader.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/DirectoryReader.java Sat Aug  4 21:26:10 2012
@@ -75,6 +75,9 @@ public abstract class DirectoryReader ex
    *  memory usage, at the expense of higher latency when
    *  loading a TermInfo.  The default value is 1.  Set this
    *  to -1 to skip loading the terms index entirely.
+   *  <b>NOTE:</b> divisor settings &gt; 1 do not apply to all PostingsFormat
+   *  implementations, including the default one in this release. It only makes
+   *  sense for terms indexes that can efficiently re-sample terms at load time.
    * @throws IOException if there is a low-level IO error
    */
   public static DirectoryReader open(final Directory directory, int termInfosIndexDivisor) throws IOException {
@@ -126,6 +129,9 @@ public abstract class DirectoryReader ex
    *  memory usage, at the expense of higher latency when
    *  loading a TermInfo.  The default value is 1.  Set this
    *  to -1 to skip loading the terms index entirely.
+   *  <b>NOTE:</b> divisor settings &gt; 1 do not apply to all PostingsFormat
+   *  implementations, including the default one in this release. It only makes
+   *  sense for terms indexes that can efficiently re-sample terms at load time.
    * @throws IOException if there is a low-level IO error
    */
   public static DirectoryReader open(final IndexCommit commit, int termInfosIndexDivisor) throws IOException {

Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java?rev=1369470&r1=1369469&r2=1369470&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java Sat Aug  4 21:26:10 2012
@@ -202,11 +202,9 @@ final class DocumentsWriter {
    *  discarding any docs added since last flush. */
   synchronized void abort() {
     boolean success = false;
-    synchronized (this) {
-      deleteQueue.clear();
-    }
 
     try {
+      deleteQueue.clear();
       if (infoStream.isEnabled("DW")) {
         infoStream.message("DW", "abort");
       }
@@ -230,6 +228,7 @@ final class DocumentsWriter {
           perThread.unlock();
         }
       }
+      flushControl.abortPendingFlushes();
       flushControl.waitForFlush();
       success = true;
     } finally {

Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterFlushControl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterFlushControl.java?rev=1369470&r1=1369469&r2=1369470&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterFlushControl.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterFlushControl.java Sat Aug  4 21:26:10 2012
@@ -567,19 +567,34 @@ final class DocumentsWriterFlushControl 
   }
 
   synchronized void abortFullFlushes() {
+   try {
+     abortPendingFlushes();
+   } finally {
+     fullFlush = false;
+   }
+  }
+  
+  synchronized void abortPendingFlushes() {
     try {
       for (DocumentsWriterPerThread dwpt : flushQueue) {
-        doAfterFlush(dwpt);
-        dwpt.abort();
+        try {
+          dwpt.abort();
+          doAfterFlush(dwpt);
+        } catch (Throwable ex) {
+          // ignore - keep on aborting the flush queue
+        }
       }
       for (BlockedFlush blockedFlush : blockedFlushes) {
-        flushingWriters
-            .put(blockedFlush.dwpt, Long.valueOf(blockedFlush.bytes));
-        doAfterFlush(blockedFlush.dwpt);
-        blockedFlush.dwpt.abort();
+        try {
+          flushingWriters
+              .put(blockedFlush.dwpt, Long.valueOf(blockedFlush.bytes));
+          blockedFlush.dwpt.abort();
+          doAfterFlush(blockedFlush.dwpt);
+        } catch (Throwable ex) {
+          // ignore - keep on aborting the blocked queue
+        }
       }
     } finally {
-      fullFlush = false;
       flushQueue.clear();
       blockedFlushes.clear();
       updateStallState();

Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/IndexReaderContext.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/IndexReaderContext.java?rev=1369470&r1=1369469&r2=1369470&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/IndexReaderContext.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/IndexReaderContext.java Sat Aug  4 21:26:10 2012
@@ -1,7 +1,5 @@
 package org.apache.lucene.index;
 
-import java.util.List;
-
 /*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
@@ -19,6 +17,8 @@ import java.util.List;
  * limitations under the License.
  */
 
+import java.util.List;
+
 /**
  * A struct like class that represents a hierarchical relationship between
  * {@link IndexReader} instances. 

Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java?rev=1369470&r1=1369469&r2=1369470&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java Sat Aug  4 21:26:10 2012
@@ -1241,6 +1241,78 @@ public class IndexWriter implements Clos
     }
   }
 
+  /** Expert: attempts to delete by document ID, as long as
+   *  the provided reader is a near-real-time reader (from {@link
+   *  DirectoryReader#open(IndexWriter,boolean)}).  If the
+   *  provided reader is an NRT reader obtained from this
+   *  writer, and its segment has not been merged away, then
+   *  the delete succeeds and this method returns true; else, it
+   *  returns false the caller must then separately delete by
+   *  Term or Query.
+   *
+   *  <b>NOTE</b>: this method can only delete documents
+   *  visible to the currently open NRT reader.  If you need
+   *  to delete documents indexed after opening the NRT
+   *  reader you must use the other deleteDocument methods
+   *  (e.g., {@link #deleteDocuments(Term)}). */
+  public synchronized boolean tryDeleteDocument(IndexReader readerIn, int docID) throws IOException {
+
+    final AtomicReader reader;
+    if (readerIn instanceof AtomicReader) {
+      // Reader is already atomic: use the incoming docID:
+      reader = (AtomicReader) readerIn;
+    } else {
+      // Composite reader: lookup sub-reader and re-base docID:
+      List<AtomicReaderContext> leaves = readerIn.getTopReaderContext().leaves();
+      int subIndex = ReaderUtil.subIndex(docID, leaves);
+      reader = leaves.get(subIndex).reader();
+      docID -= leaves.get(subIndex).docBase;
+      assert docID >= 0;
+      assert docID < reader.maxDoc();
+    }
+
+    if (!(reader instanceof SegmentReader)) {
+      throw new IllegalArgumentException("the reader must be a SegmentReader or composite reader containing only SegmentReaders");
+    }
+      
+    final SegmentInfoPerCommit info = ((SegmentReader) reader).getSegmentInfo();
+
+    // TODO: this is a slow linear search, but, number of
+    // segments should be contained unless something is
+    // seriously wrong w/ the index, so it should be a minor
+    // cost:
+
+    if (segmentInfos.indexOf(info) != -1) {
+      ReadersAndLiveDocs rld = readerPool.get(info, false);
+      if (rld != null) {
+        synchronized(bufferedDeletesStream) {
+          rld.initWritableLiveDocs();
+          if (rld.delete(docID)) {
+            final int fullDelCount = rld.info.getDelCount() + rld.getPendingDeleteCount();
+            if (fullDelCount == rld.info.info.getDocCount()) {
+              // If a merge has already registered for this
+              // segment, we leave it in the readerPool; the
+              // merge will skip merging it and will then drop
+              // it once it's done:
+              if (!mergingSegments.contains(rld.info)) {
+                segmentInfos.remove(rld.info);
+                readerPool.drop(rld.info);
+                checkpoint();
+              }
+            }
+          }
+          //System.out.println("  yes " + info.info.name + " " + docID);
+          return true;
+        }
+      } else {
+        //System.out.println("  no rld " + info.info.name + " " + docID);
+      }
+    } else {
+      //System.out.println("  no seg " + info.info.name + " " + docID);
+    }
+    return false;
+  }
+
   /**
    * Deletes the document(s) containing any of the
    * terms. All given deletes are applied and flushed atomically

Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/IndexWriterConfig.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/IndexWriterConfig.java?rev=1369470&r1=1369469&r2=1369470&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/IndexWriterConfig.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/IndexWriterConfig.java Sat Aug  4 21:26:10 2012
@@ -392,7 +392,7 @@ public final class IndexWriterConfig ext
    * @see #setMaxBufferedDocs(int)
    * @see #setRAMBufferSizeMB(double)
    */
-  public IndexWriterConfig setFlushPolicy(FlushPolicy flushPolicy) {
+  IndexWriterConfig setFlushPolicy(FlushPolicy flushPolicy) {
     this.flushPolicy = flushPolicy;
     return this;
   }
@@ -422,7 +422,7 @@ public final class IndexWriterConfig ext
   }
   
   @Override
-  public FlushPolicy getFlushPolicy() {
+  FlushPolicy getFlushPolicy() {
     return flushPolicy;
   }
   

Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/LiveIndexWriterConfig.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/LiveIndexWriterConfig.java?rev=1369470&r1=1369469&r2=1369470&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/LiveIndexWriterConfig.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/LiveIndexWriterConfig.java Sat Aug  4 21:26:10 2012
@@ -19,6 +19,7 @@ package org.apache.lucene.index;
 
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.codecs.Codec;
+import org.apache.lucene.codecs.lucene40.Lucene40PostingsFormat; // javadocs
 import org.apache.lucene.index.DocumentsWriterPerThread.IndexingChain;
 import org.apache.lucene.index.IndexWriter.IndexReaderWarmer;
 import org.apache.lucene.index.IndexWriterConfig.OpenMode;
@@ -146,6 +147,29 @@ public class LiveIndexWriterConfig {
    * Takes effect immediately, but only applies to newly flushed/merged
    * segments.
    * 
+   * <p>
+   * <b>NOTE:</b> This parameter does not apply to all PostingsFormat implementations,
+   * including the default one in this release. It only makes sense for term indexes
+   * that are implemented as a fixed gap between terms. For example, 
+   * {@link Lucene40PostingsFormat} implements the term index instead based upon how
+   * terms share prefixes. To configure its parameters (the minimum and maximum size
+   * for a block), you would instead use  {@link Lucene40PostingsFormat#Lucene40PostingsFormat(int, int)}.
+   * which can also be configured on a per-field basis:
+   * <pre class="prettyprint">
+   * //customize Lucene40PostingsFormat, passing minBlockSize=50, maxBlockSize=100
+   * final PostingsFormat tweakedPostings = new Lucene40PostingsFormat(50, 100);
+   * iwc.setCodec(new Lucene40Codec() {
+   *   &#64;Override
+   *   public PostingsFormat getPostingsFormatForField(String field) {
+   *     if (field.equals("fieldWithTonsOfTerms"))
+   *       return tweakedPostings;
+   *     else
+   *       return super.getPostingsFormatForField(field);
+   *   }
+   * });
+   * </pre>
+   * Note that other implementations may have their own parameters, or no parameters at all.
+   * 
    * @see IndexWriterConfig#DEFAULT_TERM_INDEX_INTERVAL
    */
   public LiveIndexWriterConfig setTermIndexInterval(int interval) { // TODO: this should be private to the codec, not settable here
@@ -335,6 +359,10 @@ public class LiveIndexWriterConfig {
    * <p>
    * Takes effect immediately, but only applies to readers opened after this
    * call
+   * <p>
+   * <b>NOTE:</b> divisor settings &gt; 1 do not apply to all PostingsFormat
+   * implementations, including the default one in this release. It only makes
+   * sense for terms indexes that can efficiently re-sample terms at load time.
    */
   public LiveIndexWriterConfig setReaderTermsIndexDivisor(int divisor) {
     if (divisor <= 0 && divisor != -1) {
@@ -462,7 +490,7 @@ public class LiveIndexWriterConfig {
   /**
    * @see IndexWriterConfig#setFlushPolicy(FlushPolicy)
    */
-  public FlushPolicy getFlushPolicy() {
+  FlushPolicy getFlushPolicy() {
     return flushPolicy;
   }
   
@@ -497,7 +525,6 @@ public class LiveIndexWriterConfig {
     sb.append("mergePolicy=").append(getMergePolicy()).append("\n");
     sb.append("indexerThreadPool=").append(getIndexerThreadPool()).append("\n");
     sb.append("readerPooling=").append(getReaderPooling()).append("\n");
-    sb.append("flushPolicy=").append(getFlushPolicy()).append("\n");
     sb.append("perThreadHardLimitMB=").append(getRAMPerThreadHardLimitMB()).append("\n");
     return sb.toString();
   }

Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/FuzzyTermsEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/FuzzyTermsEnum.java?rev=1369470&r1=1369469&r2=1369470&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/FuzzyTermsEnum.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/FuzzyTermsEnum.java Sat Aug  4 21:26:10 2012
@@ -122,7 +122,7 @@ public class FuzzyTermsEnum extends Term
     this.realPrefixLength = prefixLength > termLength ? termLength : prefixLength;
     // if minSimilarity >= 1, we treat it as number of edits
     if (minSimilarity >= 1f) {
-      this.minSimilarity = 1 - (minSimilarity+1) / this.termLength;
+      this.minSimilarity = 0; // just driven by number of edits
       maxEdits = (int) minSimilarity;
       raw = true;
     } else {

Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/NRTManager.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/NRTManager.java?rev=1369470&r1=1369469&r2=1369470&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/NRTManager.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/NRTManager.java Sat Aug  4 21:26:10 2012
@@ -27,6 +27,7 @@ import java.util.concurrent.locks.Reentr
 
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.SegmentInfoPerCommit;
 import org.apache.lucene.index.IndexReader; // javadocs
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.IndexableField;
@@ -254,6 +255,14 @@ public class NRTManager extends Referenc
     long getAndIncrementGeneration() {
       return indexingGen.getAndIncrement();
     }
+
+    public long tryDeleteDocument(IndexReader reader, int docID) throws IOException {
+      if (writer.tryDeleteDocument(reader, docID)) {
+        return indexingGen.get();
+      } else {
+        return -1;
+      }
+    }
   }
 
   /**

Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/util/FixedBitSet.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/util/FixedBitSet.java?rev=1369470&r1=1369469&r2=1369470&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/util/FixedBitSet.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/util/FixedBitSet.java Sat Aug  4 21:26:10 2012
@@ -54,6 +54,11 @@ public final class FixedBitSet extends D
     bits = new long[bits2words(numBits)];
   }
 
+  public FixedBitSet(long[]storedBits,int numBits) {
+    this.numBits = numBits;
+    this.bits = storedBits;
+  }      
+  
   /** Makes full copy. */
   public FixedBitSet(FixedBitSet other) {
     bits = new long[other.bits.length];

Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat?rev=1369470&r1=1369469&r2=1369470&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat Sat Aug  4 21:26:10 2012
@@ -18,6 +18,7 @@ org.apache.lucene.codecs.pulsing.Pulsing
 org.apache.lucene.codecs.simpletext.SimpleTextPostingsFormat
 org.apache.lucene.codecs.memory.MemoryPostingsFormat
 org.apache.lucene.codecs.bulkvint.BulkVIntPostingsFormat
+org.apache.lucene.codecs.bloom.BloomFilteringPostingsFormat
+org.apache.lucene.codecs.memory.DirectPostingsFormat
 org.apache.lucene.codecs.block.BlockPostingsFormat
 org.apache.lucene.codecs.blockpacked.BlockPackedPostingsFormat
-org.apache.lucene.codecs.memory.DirectPostingsFormat

Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/document/TestDocument.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/document/TestDocument.java?rev=1369470&r1=1369469&r2=1369470&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/document/TestDocument.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/document/TestDocument.java Sat Aug  4 21:26:10 2012
@@ -290,33 +290,4 @@ public class TestDocument extends Lucene
       // expected
     }
   }
-
-  public void testBoost() throws Exception {
-    Directory dir = newDirectory();
-    IndexWriterConfig iwc = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
-    iwc.setMergePolicy(newLogMergePolicy());
-    IndexWriter iw = new IndexWriter(dir, iwc);
-    Document doc = new Document();
-    doc.add(new StringField("field1", "sometext", Field.Store.YES));
-    doc.add(new TextField("field2", "sometext", Field.Store.NO));
-    doc.add(new StringField("foo", "bar", Field.Store.NO));
-    iw.addDocument(doc); // add an 'ok' document
-    try {
-      doc = new Document();
-      // try to boost with norms omitted
-      StringField field = new StringField("foo", "baz", Field.Store.NO);
-      field.setBoost(5.0f);
-      doc.add(field);
-      iw.addDocument(doc);
-      fail("didn't get any exception, boost silently discarded");
-    } catch (UnsupportedOperationException expected) {
-      // expected
-    }
-    DirectoryReader ir = DirectoryReader.open(iw, false);
-    assertEquals(1, ir.numDocs());
-    assertEquals("sometext", ir.document(0).get("field1"));
-    ir.close();
-    iw.close();
-    dir.close();
-  }
 }

Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestByteSlices.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestByteSlices.java?rev=1369470&r1=1369469&r2=1369470&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestByteSlices.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestByteSlices.java Sat Aug  4 21:26:10 2012
@@ -40,24 +40,41 @@ public class TestByteSlices extends Luce
         counters[stream] = 0;
       }
       
-      int num = atLeast(10000);
+      int num = atLeast(3000);
       for (int iter = 0; iter < num; iter++) {
-        int stream = random().nextInt(NUM_STREAM);
-        if (VERBOSE)
+        int stream;
+        if (random().nextBoolean()) {
+          stream = random().nextInt(3);
+        } else {
+          stream = random().nextInt(NUM_STREAM);
+        }
+
+        if (VERBOSE) {
           System.out.println("write stream=" + stream);
+        }
 
         if (starts[stream] == -1) {
           final int spot = pool.newSlice(ByteBlockPool.FIRST_LEVEL_SIZE);
           starts[stream] = uptos[stream] = spot + pool.byteOffset;
-          if (VERBOSE)
+          if (VERBOSE) {
             System.out.println("  init to " + starts[stream]);
+          }
         }
 
         writer.init(uptos[stream]);
-        int numValue = random().nextInt(20);
+        int numValue;
+        if (random().nextInt(10) == 3) {
+          numValue = random().nextInt(100);
+        } else if (random().nextInt(5) == 3) {
+          numValue = random().nextInt(3);
+        } else {
+          numValue = random().nextInt(20);
+        }
+
         for(int j=0;j<numValue;j++) {
-          if (VERBOSE)
+          if (VERBOSE) {
             System.out.println("    write " + (counters[stream]+j));
+          }
           // write some large (incl. negative) ints:
           writer.writeVInt(random().nextInt());
           writer.writeVInt(counters[stream]+j);

Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestDoc.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestDoc.java?rev=1369470&r1=1369469&r2=1369470&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestDoc.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestDoc.java Sat Aug  4 21:26:10 2012
@@ -186,7 +186,7 @@ public class TestDoc extends LuceneTestC
       File file = new File(workDir, fileName);
       Document doc = new Document();
       InputStreamReader is = new InputStreamReader(new FileInputStream(file), "UTF-8");
-      doc.add(new TextField("contents", is, Field.Store.NO));
+      doc.add(new TextField("contents", is));
       writer.addDocument(doc);
       writer.commit();
       is.close();

Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterDelete.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterDelete.java?rev=1369470&r1=1369469&r2=1369470&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterDelete.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterDelete.java Sat Aug  4 21:26:10 2012
@@ -891,9 +891,12 @@ public class TestIndexWriterDelete exten
   }
   
   public void testIndexingThenDeleting() throws Exception {
+    // TODO: move this test to its own class and just @SuppressCodecs?
+    // TODO: is it enough to just use newFSDirectory?
     final String fieldFormat = _TestUtil.getPostingsFormat("field");
     assumeFalse("This test cannot run with Memory codec", fieldFormat.equals("Memory"));
     assumeFalse("This test cannot run with SimpleText codec", fieldFormat.equals("SimpleText"));
+    assumeFalse("This test cannot run with Direct codec", fieldFormat.equals("Direct"));
     final Random r = random();
     Directory dir = newDirectory();
     // note this test explicitly disables payloads

Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java?rev=1369470&r1=1369469&r2=1369470&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java Sat Aug  4 21:26:10 2012
@@ -1542,4 +1542,73 @@ public class TestIndexWriterExceptions e
     iw.close();
     dir.close();
   }
+  
+  public void testBoostOmitNorms() throws Exception {
+    Directory dir = newDirectory();
+    IndexWriterConfig iwc = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
+    iwc.setMergePolicy(newLogMergePolicy());
+    IndexWriter iw = new IndexWriter(dir, iwc);
+    Document doc = new Document();
+    doc.add(new StringField("field1", "sometext", Field.Store.YES));
+    doc.add(new TextField("field2", "sometext", Field.Store.NO));
+    doc.add(new StringField("foo", "bar", Field.Store.NO));
+    iw.addDocument(doc); // add an 'ok' document
+    try {
+      doc = new Document();
+      // try to boost with norms omitted
+      List<IndexableField> list = new ArrayList<IndexableField>();
+      list.add(new IndexableField() {
+
+        @Override
+        public String name() {
+          return "foo";
+        }
+
+        @Override
+        public IndexableFieldType fieldType() {
+          return StringField.TYPE_NOT_STORED;
+        }
+
+        @Override
+        public float boost() {
+          return 5f;
+        }
+
+        @Override
+        public BytesRef binaryValue() {
+          return null;
+        }
+
+        @Override
+        public String stringValue() {
+          return "baz";
+        }
+
+        @Override
+        public Reader readerValue() {
+          return null;
+        }
+
+        @Override
+        public Number numericValue() {
+          return null;
+        }
+
+        @Override
+        public TokenStream tokenStream(Analyzer analyzer) throws IOException {
+          return null;
+        }
+      });
+      iw.addDocument(list);
+      fail("didn't get any exception, boost silently discarded");
+    } catch (UnsupportedOperationException expected) {
+      // expected
+    }
+    DirectoryReader ir = DirectoryReader.open(iw, false);
+    assertEquals(1, ir.numDocs());
+    assertEquals("sometext", ir.document(0).get("field1"));
+    ir.close();
+    iw.close();
+    dir.close();
+  }
 }

Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestRollingUpdates.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestRollingUpdates.java?rev=1369470&r1=1369469&r2=1369470&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestRollingUpdates.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestRollingUpdates.java Sat Aug  4 21:26:10 2012
@@ -23,6 +23,9 @@ import org.apache.lucene.analysis.MockAn
 import org.apache.lucene.codecs.Codec;
 import org.apache.lucene.codecs.memory.MemoryPostingsFormat;
 import org.apache.lucene.document.*;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.TopDocs;
 import org.apache.lucene.store.*;
 import org.apache.lucene.util.*;
 import org.junit.Test;
@@ -48,10 +51,13 @@ public class TestRollingUpdates extends 
     final int SIZE = atLeast(20);
     int id = 0;
     IndexReader r = null;
+    IndexSearcher s = null;
     final int numUpdates = (int) (SIZE * (2+(TEST_NIGHTLY ? 200*random().nextDouble() : 5*random().nextDouble())));
     if (VERBOSE) {
       System.out.println("TEST: numUpdates=" + numUpdates);
     }
+    int updateCount = 0;
+    // TODO: sometimes update ids not in order...
     for(int docIter=0;docIter<numUpdates;docIter++) {
       final Document doc = docs.nextDoc();
       final String myID = ""+id;
@@ -60,16 +66,59 @@ public class TestRollingUpdates extends 
       } else {
         id++;
       }
+      if (VERBOSE) {
+        System.out.println("  docIter=" + docIter + " id=" + id);
+      }
       ((Field) doc.getField("docid")).setStringValue(myID);
-      w.updateDocument(new Term("docid", myID), doc);
+
+      Term idTerm = new Term("docid", myID);
+
+      final boolean doUpdate;
+      if (s != null && updateCount < SIZE) {
+        TopDocs hits = s.search(new TermQuery(idTerm), 1);
+        assertEquals(1, hits.totalHits);
+        doUpdate = !w.tryDeleteDocument(r, hits.scoreDocs[0].doc);
+        if (VERBOSE) {
+          if (doUpdate) {
+            System.out.println("  tryDeleteDocument failed");
+          } else {
+            System.out.println("  tryDeleteDocument succeeded");
+          }
+        }
+      } else {
+        doUpdate = true;
+        if (VERBOSE) {
+          System.out.println("  no searcher: doUpdate=true");
+        }
+      }
+
+      updateCount++;
+
+      if (doUpdate) {
+        w.updateDocument(idTerm, doc);
+      } else {
+        w.addDocument(doc);
+      }
 
       if (docIter >= SIZE && random().nextInt(50) == 17) {
         if (r != null) {
           r.close();
         }
+
         final boolean applyDeletions = random().nextBoolean();
+
+        if (VERBOSE) {
+          System.out.println("TEST: reopen applyDeletions=" + applyDeletions);
+        }
+
         r = w.getReader(applyDeletions);
+        if (applyDeletions) {
+          s = new IndexSearcher(r);
+        } else {
+          s = null;
+        }
         assertTrue("applyDeletions=" + applyDeletions + " r.numDocs()=" + r.numDocs() + " vs SIZE=" + SIZE, !applyDeletions || r.numDocs() == SIZE);
+        updateCount = 0;
       }
     }
 

Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/search/TestFuzzyQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/search/TestFuzzyQuery.java?rev=1369470&r1=1369469&r2=1369470&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/search/TestFuzzyQuery.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/search/TestFuzzyQuery.java Sat Aug  4 21:26:10 2012
@@ -22,6 +22,7 @@ import java.util.Arrays;
 import java.io.IOException;
 
 import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.analysis.MockTokenizer;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.index.IndexReader;
@@ -189,6 +190,41 @@ public class TestFuzzyQuery extends Luce
     directory.close();
   }
   
+  public void test2() throws Exception {
+    Directory directory = newDirectory();
+    RandomIndexWriter writer = new RandomIndexWriter(random(), directory, new MockAnalyzer(random(), MockTokenizer.KEYWORD, false));
+    addDoc("LANGE", writer);
+    addDoc("LUETH", writer);
+    addDoc("PIRSING", writer);
+    addDoc("RIEGEL", writer);
+    addDoc("TRZECZIAK", writer);
+    addDoc("WALKER", writer);
+    addDoc("WBR", writer);
+    addDoc("WE", writer);
+    addDoc("WEB", writer);
+    addDoc("WEBE", writer);
+    addDoc("WEBER", writer);
+    addDoc("WEBERE", writer);
+    addDoc("WEBREE", writer);
+    addDoc("WEBEREI", writer);
+    addDoc("WBRE", writer);
+    addDoc("WITTKOPF", writer);
+    addDoc("WOJNAROWSKI", writer);
+    addDoc("WRICKE", writer);
+
+    IndexReader reader = writer.getReader();
+    IndexSearcher searcher = newSearcher(reader);
+    writer.close();
+
+    FuzzyQuery query = new FuzzyQuery(new Term("field", "WEBER"), 2, 1);
+    //query.setRewriteMethod(FuzzyQuery.SCORING_BOOLEAN_QUERY_REWRITE);
+    ScoreDoc[] hits = searcher.search(query, null, 1000).scoreDocs;
+    assertEquals(8, hits.length);
+
+    reader.close();
+    directory.close();
+  }
+  
   /** 
    * MultiTermQuery provides (via attribute) information about which values
    * must be competitive to enter the priority queue. 

Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/search/TestPositionIncrement.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/search/TestPositionIncrement.java?rev=1369470&r1=1369469&r2=1369470&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/search/TestPositionIncrement.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/search/TestPositionIncrement.java Sat Aug  4 21:26:10 2012
@@ -206,7 +206,7 @@ public class TestPositionIncrement exten
     RandomIndexWriter writer = new RandomIndexWriter(random(), dir, new MockPayloadAnalyzer());
     Document doc = new Document();
     doc.add(new TextField("content", new StringReader(
-        "a a b c d e a f g h i j a b k k"), Field.Store.NO));
+        "a a b c d e a f g h i j a b k k")));
     writer.addDocument(doc);
 
     final IndexReader readerFromWriter = writer.getReader();

Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/search/TestShardSearching.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/search/TestShardSearching.java?rev=1369470&r1=1369469&r2=1369470&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/search/TestShardSearching.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/search/TestShardSearching.java Sat Aug  4 21:26:10 2012
@@ -77,8 +77,7 @@ public class TestShardSearching extends 
       System.out.println("TEST: numNodes=" + numNodes + " runTimeSec=" + runTimeSec + " maxSearcherAgeSeconds=" + maxSearcherAgeSeconds);
     }
 
-    start(_TestUtil.getTempDir("TestShardSearching").toString(),
-          numNodes,
+    start(numNodes,
           runTimeSec,
           maxSearcherAgeSeconds
           );

Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/search/TestSort.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/search/TestSort.java?rev=1369470&r1=1369469&r2=1369470&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/search/TestSort.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/search/TestSort.java Sat Aug  4 21:26:10 2012
@@ -196,7 +196,7 @@ public class TestSort extends LuceneTest
         if (data[i][11] != null) doc.add (new StringField ("parser",     data[i][11], Field.Store.NO));
 
         for(IndexableField f : doc.getFields()) {
-          if (!f.fieldType().omitNorms()) {
+          if (f.fieldType().indexed() && !f.fieldType().omitNorms()) {
             ((Field) f).setBoost(2.0f);
           }
         }
@@ -239,7 +239,7 @@ public class TestSort extends LuceneTest
       doc.add(new SortedBytesDocValuesField("string2", new BytesRef(num2)));
       doc.add (new Field ("tracer2", num2, onlyStored));
       for(IndexableField f2 : doc.getFields()) {
-        if (!f2.fieldType().omitNorms()) {
+        if (f2.fieldType().indexed() && !f2.fieldType().omitNorms()) {
           ((Field) f2).setBoost(2.0f);
         }
       }
@@ -255,7 +255,7 @@ public class TestSort extends LuceneTest
       doc.add (new Field ("tracer2_fixed", num2Fixed, onlyStored));
 
       for(IndexableField f2 : doc.getFields()) {
-        if (!f2.fieldType().omitNorms()) {
+        if (f2.fieldType().indexed() && !f2.fieldType().omitNorms()) {
           ((Field) f2).setBoost(2.0f);
         }
       }

Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/search/spans/TestPayloadSpans.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/search/spans/TestPayloadSpans.java?rev=1369470&r1=1369469&r2=1369470&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/search/spans/TestPayloadSpans.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/search/spans/TestPayloadSpans.java Sat Aug  4 21:26:10 2012
@@ -256,7 +256,7 @@ public class TestPayloadSpans extends Lu
                                                      newIndexWriterConfig(TEST_VERSION_CURRENT, new TestPayloadAnalyzer()));
 
     Document doc = new Document();
-    doc.add(new TextField("content", new StringReader("a b c d e f g h i j a k"), Field.Store.NO));
+    doc.add(new TextField("content", new StringReader("a b c d e f g h i j a k")));
     writer.addDocument(doc);
 
     IndexReader reader = writer.getReader();
@@ -293,7 +293,7 @@ public class TestPayloadSpans extends Lu
                                                      newIndexWriterConfig(TEST_VERSION_CURRENT, new TestPayloadAnalyzer()));
 
     Document doc = new Document();
-    doc.add(new TextField("content", new StringReader("a b a d k f a h i k a k"), Field.Store.NO));
+    doc.add(new TextField("content", new StringReader("a b a d k f a h i k a k")));
     writer.addDocument(doc);
     IndexReader reader = writer.getReader();
     IndexSearcher is = newSearcher(reader);
@@ -328,7 +328,7 @@ public class TestPayloadSpans extends Lu
                                                      newIndexWriterConfig(TEST_VERSION_CURRENT, new TestPayloadAnalyzer()));
 
     Document doc = new Document();
-    doc.add(new TextField("content", new StringReader("j k a l f k k p a t a k l k t a"), Field.Store.NO));
+    doc.add(new TextField("content", new StringReader("j k a l f k k p a t a k l k t a")));
     writer.addDocument(doc);
     IndexReader reader = writer.getReader();
     IndexSearcher is = newSearcher(reader);

Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/util/fst/TestFSTs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/util/fst/TestFSTs.java?rev=1369470&r1=1369469&r2=1369470&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/util/fst/TestFSTs.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/util/fst/TestFSTs.java Sat Aug  4 21:26:10 2012
@@ -1096,13 +1096,6 @@ public class TestFSTs extends LuceneTest
   // file, up until a time limit
   public void testRealTerms() throws Exception {
 
-    // TODO: is this necessary? we use the annotation...
-    final String defaultFormat = _TestUtil.getPostingsFormat("abracadabra");
-    if (defaultFormat.equals("SimpleText") || defaultFormat.equals("Memory")) {
-      // no
-      Codec.setDefault(_TestUtil.alwaysPostingsFormat(new Lucene40PostingsFormat()));
-    }
-
     final LineFileDocs docs = new LineFileDocs(random(), true);
     final int RUN_TIME_MSEC = atLeast(500);
     final IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())).setMaxBufferedDocs(-1).setRAMBufferSizeMB(64);

Modified: lucene/dev/branches/pforcodec_3892/lucene/demo/src/java/org/apache/lucene/demo/IndexFiles.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/demo/src/java/org/apache/lucene/demo/IndexFiles.java?rev=1369470&r1=1369469&r2=1369470&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/demo/src/java/org/apache/lucene/demo/IndexFiles.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/demo/src/java/org/apache/lucene/demo/IndexFiles.java Sat Aug  4 21:26:10 2012
@@ -190,7 +190,7 @@ public class IndexFiles {
           // so that the text of the file is tokenized and indexed, but not stored.
           // Note that FileReader expects the file to be in UTF-8 encoding.
           // If that's not the case searching for special characters will fail.
-          doc.add(new TextField("contents", new BufferedReader(new InputStreamReader(fis, "UTF-8")), Field.Store.NO));
+          doc.add(new TextField("contents", new BufferedReader(new InputStreamReader(fis, "UTF-8"))));
 
           if (writer.getConfig().getOpenMode() == OpenMode.CREATE) {
             // New index, so we just add the document (no old document can be there):

Modified: lucene/dev/branches/pforcodec_3892/lucene/sandbox/src/test/org/apache/lucene/sandbox/queries/TestSlowFuzzyQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/sandbox/src/test/org/apache/lucene/sandbox/queries/TestSlowFuzzyQuery.java?rev=1369470&r1=1369469&r2=1369470&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/sandbox/src/test/org/apache/lucene/sandbox/queries/TestSlowFuzzyQuery.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/sandbox/src/test/org/apache/lucene/sandbox/queries/TestSlowFuzzyQuery.java Sat Aug  4 21:26:10 2012
@@ -440,21 +440,25 @@ public class TestSlowFuzzyQuery extends 
     assertEquals(1, hits.length);
     assertEquals("foobar", searcher.doc(hits[0].doc).get("field"));
     
-    q = new SlowFuzzyQuery(new Term("field", "t"), 3);
-    hits = searcher.search(q, 10).scoreDocs;
-    assertEquals(1, hits.length);
-    assertEquals("test", searcher.doc(hits[0].doc).get("field"));
+    // TODO: cannot really be supported given the legacy scoring
+    // system which scores negative, if the distance > min term len,
+    // so such matches were always impossible with lucene 3.x, etc
+    //
+    //q = new SlowFuzzyQuery(new Term("field", "t"), 3);
+    //hits = searcher.search(q, 10).scoreDocs;
+    //assertEquals(1, hits.length);
+    //assertEquals("test", searcher.doc(hits[0].doc).get("field"));
     
-    q = new SlowFuzzyQuery(new Term("field", "a"), 4f, 0, 50);
-    hits = searcher.search(q, 10).scoreDocs;
-    assertEquals(1, hits.length);
-    assertEquals("test", searcher.doc(hits[0].doc).get("field"));
+    // q = new SlowFuzzyQuery(new Term("field", "a"), 4f, 0, 50);
+    // hits = searcher.search(q, 10).scoreDocs;
+    // assertEquals(1, hits.length);
+    // assertEquals("test", searcher.doc(hits[0].doc).get("field"));
     
-    q = new SlowFuzzyQuery(new Term("field", "a"), 6f, 0, 50);
-    hits = searcher.search(q, 10).scoreDocs;
-    assertEquals(2, hits.length);
-    assertEquals("test", searcher.doc(hits[0].doc).get("field"));
-    assertEquals("foobar", searcher.doc(hits[1].doc).get("field"));
+    // q = new SlowFuzzyQuery(new Term("field", "a"), 6f, 0, 50);
+    // hits = searcher.search(q, 10).scoreDocs;
+    // assertEquals(2, hits.length);
+    // assertEquals("test", searcher.doc(hits[0].doc).get("field"));
+    // assertEquals("foobar", searcher.doc(hits[1].doc).get("field"));
     
     reader.close();
     index.close();

Modified: lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/index/RandomCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/index/RandomCodec.java?rev=1369470&r1=1369469&r2=1369470&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/index/RandomCodec.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/index/RandomCodec.java Sat Aug  4 21:26:10 2012
@@ -29,6 +29,7 @@ import java.util.Set;
 
 import org.apache.lucene.codecs.PostingsFormat;
 import org.apache.lucene.codecs.asserting.AssertingPostingsFormat;
+import org.apache.lucene.codecs.bloom.TestBloomFilteredLucene40Postings;
 import org.apache.lucene.codecs.lucene40.Lucene40Codec;
 import org.apache.lucene.codecs.lucene40.Lucene40PostingsFormat;
 import org.apache.lucene.codecs.lucene40ords.Lucene40WithOrds;
@@ -98,6 +99,10 @@ public class RandomCodec extends Lucene4
         new Pulsing40PostingsFormat(1 + random.nextInt(20), minItemsPerBlock, maxItemsPerBlock),
         // add pulsing again with (usually) different parameters
         new Pulsing40PostingsFormat(1 + random.nextInt(20), minItemsPerBlock, maxItemsPerBlock),
+        //TODO as a PostingsFormat which wraps others, we should allow TestBloomFilteredLucene40Postings to be constructed 
+        //with a choice of concrete PostingsFormats. Maybe useful to have a generic means of marking and dealing 
+        //with such "wrapper" classes?
+        new TestBloomFilteredLucene40Postings(),                
         new MockSepPostingsFormat(),
         new MockFixedIntBlockPostingsFormat(_TestUtil.nextInt(random, 1, 2000)),
         new MockVariableIntBlockPostingsFormat( _TestUtil.nextInt(random, 1, 127)),

Modified: lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/search/ShardSearchingTestBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/search/ShardSearchingTestBase.java?rev=1369470&r1=1369469&r2=1369470&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/search/ShardSearchingTestBase.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/search/ShardSearchingTestBase.java Sat Aug  4 21:26:10 2012
@@ -18,7 +18,6 @@ package org.apache.lucene.search;
  */
 
 import java.io.Closeable;
-import java.io.File;
 import java.io.IOException;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -27,15 +26,17 @@ import java.util.Random;
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 
+import org.apache.lucene.analysis.MockAnalyzer;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.IndexWriterConfig;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.index.TermContext;
-import org.apache.lucene.analysis.MockAnalyzer;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.util.LineFileDocs;
 import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.PrintStreamInfoStream;
+import org.apache.lucene.util._TestUtil;
 
 // TODO
 //   - doc blocks?  so we can test joins/grouping...
@@ -423,11 +424,16 @@ public abstract class ShardSearchingTest
 
     private volatile ShardIndexSearcher currentShardSearcher;
 
-    public NodeState(Random random, String baseDir, int nodeID, int numNodes) throws IOException {
+    public NodeState(Random random, int nodeID, int numNodes) throws IOException {
       myNodeID = nodeID;
-      dir = newFSDirectory(new File(baseDir + "." + myNodeID));
+      dir = newFSDirectory(_TestUtil.getTempDir("ShardSearchingTestBase"));
       // TODO: set warmer
-      writer = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)));
+      IndexWriterConfig iwc = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random));
+      iwc.setOpenMode(IndexWriterConfig.OpenMode.CREATE);
+      if (VERBOSE) {
+        iwc.setInfoStream(new PrintStreamInfoStream(System.out));
+      }
+      writer = new IndexWriter(dir, iwc);
       mgr = new SearcherManager(writer, true, null);
       searchers = new SearcherLifetimeManager();
 
@@ -556,14 +562,14 @@ public abstract class ShardSearchingTest
   long endTimeNanos;
   private Thread changeIndicesThread;
 
-  protected void start(String baseDirName, int numNodes, double runTimeSec, int maxSearcherAgeSeconds) throws IOException {
+  protected void start(int numNodes, double runTimeSec, int maxSearcherAgeSeconds) throws IOException {
 
     endTimeNanos = System.nanoTime() + (long) (runTimeSec*1000000000);
     this.maxSearcherAgeSeconds = maxSearcherAgeSeconds;
 
     nodes = new NodeState[numNodes];
     for(int nodeID=0;nodeID<numNodes;nodeID++) {
-      nodes[nodeID] = new NodeState(random(), baseDirName, nodeID, numNodes);
+      nodes[nodeID] = new NodeState(random(), nodeID, numNodes);
     }
 
     long[] nodeVersions = new long[nodes.length];

Modified: lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/store/BaseDirectoryWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/store/BaseDirectoryWrapper.java?rev=1369470&r1=1369469&r2=1369470&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/store/BaseDirectoryWrapper.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/store/BaseDirectoryWrapper.java Sat Aug  4 21:26:10 2012
@@ -31,8 +31,6 @@ import org.apache.lucene.util._TestUtil;
 public class BaseDirectoryWrapper extends Directory {
   /** our in directory */
   protected final Directory delegate;
-  /** best effort: base on in Directory is volatile */
-  protected boolean open;
   
   private boolean checkIndexOnClose = true;
   private boolean crossCheckTermVectorsOnClose = true;
@@ -43,7 +41,7 @@ public class BaseDirectoryWrapper extend
 
   @Override
   public void close() throws IOException {
-    open = false;
+    isOpen = false;
     if (checkIndexOnClose && indexPossiblyExists()) {
       _TestUtil.checkIndex(this, crossCheckTermVectorsOnClose);
     }
@@ -51,7 +49,7 @@ public class BaseDirectoryWrapper extend
   }
   
   public boolean isOpen() {
-    return open;
+    return isOpen;
   }
   
   /** 

Modified: lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java?rev=1369470&r1=1369469&r2=1369470&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java Sat Aug  4 21:26:10 2012
@@ -551,7 +551,7 @@ public class MockDirectoryWrapper extend
     if (noDeleteOpenFile && openLocks.size() > 0) {
       throw new RuntimeException("MockDirectoryWrapper: cannot close: there are still open locks: " + openLocks);
     }
-    open = false;
+    isOpen = false;
     if (getCheckIndexOnClose()) {
       if (indexPossiblyExists()) {
         if (LuceneTestCase.VERBOSE) {
@@ -614,11 +614,6 @@ public class MockDirectoryWrapper extend
   public synchronized void removeIndexInput(IndexInput in, String name) {
     removeOpenFile(in, name);
   }
-
-  @Override
-  public synchronized boolean isOpen() {
-    return open;
-  }
   
   /**
    * Objects that represent fail-able conditions. Objects of a derived

Modified: lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat?rev=1369470&r1=1369469&r2=1369470&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat Sat Aug  4 21:26:10 2012
@@ -20,5 +20,6 @@ org.apache.lucene.codecs.mocksep.MockSep
 org.apache.lucene.codecs.nestedpulsing.NestedPulsingPostingsFormat
 org.apache.lucene.codecs.ramonly.RAMOnlyPostingsFormat
 org.apache.lucene.codecs.lucene40ords.Lucene40WithOrds
+org.apache.lucene.codecs.bloom.TestBloomFilteredLucene40Postings
 org.apache.lucene.codecs.asserting.AssertingPostingsFormat
 



Mime
View raw message