Return-Path: X-Original-To: apmail-lucene-commits-archive@www.apache.org Delivered-To: apmail-lucene-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id E3A3E96BE for ; Sat, 4 Aug 2012 21:28:02 +0000 (UTC) Received: (qmail 9504 invoked by uid 500); 4 Aug 2012 21:28:02 -0000 Mailing-List: contact commits-help@lucene.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@lucene.apache.org Delivered-To: mailing list commits@lucene.apache.org Received: (qmail 9497 invoked by uid 99); 4 Aug 2012 21:28:02 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 04 Aug 2012 21:28:02 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 04 Aug 2012 21:27:59 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 6058C2388C56; Sat, 4 Aug 2012 21:26:22 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1369470 [20/23] - in /lucene/dev/branches/pforcodec_3892: ./ dev-tools/ dev-tools/eclipse/ dev-tools/maven/ dev-tools/scripts/ lucene/ lucene/analysis/ lucene/analysis/common/ lucene/analysis/common/src/java/org/tartarus/snowball/ext/ luce... Date: Sat, 04 Aug 2012 21:26:15 -0000 To: commits@lucene.apache.org From: mikemccand@apache.org X-Mailer: svnmailer-1.0.8-patched Message-Id: <20120804212622.6058C2388C56@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Modified: lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/snowball/TestSnowball.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/snowball/TestSnowball.java?rev=1369470&r1=1369469&r2=1369470&view=diff ============================================================================== --- lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/snowball/TestSnowball.java (original) +++ lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/snowball/TestSnowball.java Sat Aug 4 21:26:10 2012 @@ -25,6 +25,7 @@ import org.apache.lucene.analysis.Analyz import org.apache.lucene.analysis.MockTokenizer; import org.apache.lucene.analysis.Tokenizer; import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.Analyzer.TokenStreamComponents; import org.apache.lucene.analysis.core.KeywordTokenizer; import org.apache.lucene.analysis.standard.StandardAnalyzer; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; @@ -116,4 +117,21 @@ public class TestSnowball extends BaseTo checkOneTermReuse(a, "", ""); } } + + public void testRandomStrings() throws IOException { + for (String lang : SNOWBALL_LANGS) { + checkRandomStrings(lang); + } + } + + public void checkRandomStrings(final String snowballLanguage) throws IOException { + Analyzer a = new Analyzer() { + @Override + protected TokenStreamComponents createComponents(String fieldName, Reader reader) { + Tokenizer t = new MockTokenizer(reader); + return new TokenStreamComponents(t, new SnowballFilter(t, snowballLanguage)); + } + }; + checkRandomData(random(), a, 1000*RANDOM_MULTIPLIER); + } } \ No newline at end of file Modified: lucene/dev/branches/pforcodec_3892/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/TaskSequence.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/TaskSequence.java?rev=1369470&r1=1369469&r2=1369470&view=diff ============================================================================== --- lucene/dev/branches/pforcodec_3892/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/TaskSequence.java (original) +++ lucene/dev/branches/pforcodec_3892/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/TaskSequence.java Sat Aug 4 21:26:10 2012 @@ -331,7 +331,9 @@ public class TaskSequence extends PerfTa // Forwards top request to children if (runningParallelTasks != null) { for(ParallelTask t : runningParallelTasks) { - t.task.stopNow(); + if (t != null) { + t.task.stopNow(); + } } } } @@ -355,6 +357,12 @@ public class TaskSequence extends PerfTa // run threads startThreads(t); + if (stopNow) { + for (ParallelTask task : t) { + task.task.stopNow(); + } + } + // wait for all threads to complete int count = 0; for (int i = 0; i < t.length; i++) { Modified: lucene/dev/branches/pforcodec_3892/lucene/build.xml URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/build.xml?rev=1369470&r1=1369469&r2=1369470&view=diff ============================================================================== --- lucene/dev/branches/pforcodec_3892/lucene/build.xml (original) +++ lucene/dev/branches/pforcodec_3892/lucene/build.xml Sat Aug 4 21:26:10 2012 @@ -35,10 +35,9 @@ MIGRATE.txt,JRE_VERSION_MIGRATION.txt, CHANGES.txt, **/lib/*.jar, - **/lib/*LICENSE*.txt, - **/lib/*NOTICE*.txt, + licenses/**, */docs/,**/README*" - excludes="build/**,site/**" + excludes="build/**,site/**,tools/**" /> @@ -572,4 +571,8 @@ + + + + Modified: lucene/dev/branches/pforcodec_3892/lucene/common-build.xml URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/common-build.xml?rev=1369470&r1=1369469&r2=1369470&view=diff ============================================================================== --- lucene/dev/branches/pforcodec_3892/lucene/common-build.xml (original) +++ lucene/dev/branches/pforcodec_3892/lucene/common-build.xml Sat Aug 4 21:26:10 2012 @@ -1567,6 +1567,43 @@ ${tests-output}/junit4-*.suites - pe + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/PostingsFormat.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/PostingsFormat.java?rev=1369470&r1=1369469&r2=1369470&view=diff ============================================================================== --- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/PostingsFormat.java (original) +++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/PostingsFormat.java Sat Aug 4 21:26:10 2012 @@ -53,7 +53,13 @@ public abstract class PostingsFormat imp /** Reads a segment. NOTE: by the time this call * returns, it must hold open any files it will need to - * use; else, those files may be deleted. */ + * use; else, those files may be deleted. + * Additionally, required files may be deleted during the execution of + * this call before there is a chance to open them. Under these + * circumstances an IOException should be thrown by the implementation. + * IOExceptions are expected and will automatically cause a retry of the + * segment opening logic with the newly revised segments. + * */ public abstract FieldsProducer fieldsProducer(SegmentReadState state) throws IOException; @Override Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/document/Field.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/document/Field.java?rev=1369470&r1=1369469&r2=1369470&view=diff ============================================================================== --- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/document/Field.java (original) +++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/document/Field.java Sat Aug 4 21:26:10 2012 @@ -377,6 +377,11 @@ public class Field implements IndexableF * @see org.apache.lucene.search.similarities.DefaultSimilarity#encodeNormValue(float) */ public void setBoost(float boost) { + if (boost != 1.0f) { + if (type.indexed() == false || type.omitNorms()) { + throw new IllegalArgumentException("You cannot set an index-time boost on an unindexed field, or one that omits norms"); + } + } this.boost = boost; } Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/document/StoredField.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/document/StoredField.java?rev=1369470&r1=1369469&r2=1369470&view=diff ============================================================================== --- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/document/StoredField.java (original) +++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/document/StoredField.java Sat Aug 4 21:26:10 2012 @@ -49,6 +49,7 @@ public final class StoredField extends F super(name, value, TYPE); } + // TODO: not great but maybe not a big problem? public StoredField(String name, int value) { super(name, TYPE); fieldsData = value; Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/document/StringField.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/document/StringField.java?rev=1369470&r1=1369469&r2=1369470&view=diff ============================================================================== --- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/document/StringField.java (original) +++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/document/StringField.java Sat Aug 4 21:26:10 2012 @@ -54,9 +54,4 @@ public final class StringField extends F public StringField(String name, String value, Store stored) { super(name, value, stored == Store.YES ? TYPE_STORED : TYPE_NOT_STORED); } - - @Override - public String stringValue() { - return (fieldsData == null) ? null : fieldsData.toString(); - } } Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/document/TextField.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/document/TextField.java?rev=1369470&r1=1369469&r2=1369470&view=diff ============================================================================== --- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/document/TextField.java (original) +++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/document/TextField.java Sat Aug 4 21:26:10 2012 @@ -46,9 +46,9 @@ public final class TextField extends Fie // TODO: add sugar for term vectors...? - /** Creates a new TextField with Reader value. */ - public TextField(String name, Reader reader, Store store) { - super(name, reader, store == Store.YES ? TYPE_STORED : TYPE_NOT_STORED); + /** Creates a new un-stored TextField with Reader value. */ + public TextField(String name, Reader reader) { + super(name, reader, TYPE_NOT_STORED); } /** Creates a new TextField with String value. */ Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/AtomicReaderContext.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/AtomicReaderContext.java?rev=1369470&r1=1369469&r2=1369470&view=diff ============================================================================== --- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/AtomicReaderContext.java (original) +++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/AtomicReaderContext.java Sat Aug 4 21:26:10 2012 @@ -1,8 +1,5 @@ package org.apache.lucene.index; -import java.util.Collections; -import java.util.List; - /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with @@ -20,6 +17,9 @@ import java.util.List; * limitations under the License. */ +import java.util.Collections; +import java.util.List; + /** * {@link IndexReaderContext} for {@link AtomicReader} instances * @lucene.experimental @@ -51,8 +51,9 @@ public final class AtomicReaderContext e @Override public List leaves() { - if (!isTopLevel) + if (!isTopLevel) { throw new UnsupportedOperationException("This is not a top-level context."); + } assert leaves != null; return leaves; } Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/DirectoryReader.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/DirectoryReader.java?rev=1369470&r1=1369469&r2=1369470&view=diff ============================================================================== --- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/DirectoryReader.java (original) +++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/DirectoryReader.java Sat Aug 4 21:26:10 2012 @@ -75,6 +75,9 @@ public abstract class DirectoryReader ex * memory usage, at the expense of higher latency when * loading a TermInfo. The default value is 1. Set this * to -1 to skip loading the terms index entirely. + * NOTE: divisor settings > 1 do not apply to all PostingsFormat + * implementations, including the default one in this release. It only makes + * sense for terms indexes that can efficiently re-sample terms at load time. * @throws IOException if there is a low-level IO error */ public static DirectoryReader open(final Directory directory, int termInfosIndexDivisor) throws IOException { @@ -126,6 +129,9 @@ public abstract class DirectoryReader ex * memory usage, at the expense of higher latency when * loading a TermInfo. The default value is 1. Set this * to -1 to skip loading the terms index entirely. + * NOTE: divisor settings > 1 do not apply to all PostingsFormat + * implementations, including the default one in this release. It only makes + * sense for terms indexes that can efficiently re-sample terms at load time. * @throws IOException if there is a low-level IO error */ public static DirectoryReader open(final IndexCommit commit, int termInfosIndexDivisor) throws IOException { Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java?rev=1369470&r1=1369469&r2=1369470&view=diff ============================================================================== --- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java (original) +++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java Sat Aug 4 21:26:10 2012 @@ -202,11 +202,9 @@ final class DocumentsWriter { * discarding any docs added since last flush. */ synchronized void abort() { boolean success = false; - synchronized (this) { - deleteQueue.clear(); - } try { + deleteQueue.clear(); if (infoStream.isEnabled("DW")) { infoStream.message("DW", "abort"); } @@ -230,6 +228,7 @@ final class DocumentsWriter { perThread.unlock(); } } + flushControl.abortPendingFlushes(); flushControl.waitForFlush(); success = true; } finally { Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterFlushControl.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterFlushControl.java?rev=1369470&r1=1369469&r2=1369470&view=diff ============================================================================== --- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterFlushControl.java (original) +++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterFlushControl.java Sat Aug 4 21:26:10 2012 @@ -567,19 +567,34 @@ final class DocumentsWriterFlushControl } synchronized void abortFullFlushes() { + try { + abortPendingFlushes(); + } finally { + fullFlush = false; + } + } + + synchronized void abortPendingFlushes() { try { for (DocumentsWriterPerThread dwpt : flushQueue) { - doAfterFlush(dwpt); - dwpt.abort(); + try { + dwpt.abort(); + doAfterFlush(dwpt); + } catch (Throwable ex) { + // ignore - keep on aborting the flush queue + } } for (BlockedFlush blockedFlush : blockedFlushes) { - flushingWriters - .put(blockedFlush.dwpt, Long.valueOf(blockedFlush.bytes)); - doAfterFlush(blockedFlush.dwpt); - blockedFlush.dwpt.abort(); + try { + flushingWriters + .put(blockedFlush.dwpt, Long.valueOf(blockedFlush.bytes)); + blockedFlush.dwpt.abort(); + doAfterFlush(blockedFlush.dwpt); + } catch (Throwable ex) { + // ignore - keep on aborting the blocked queue + } } } finally { - fullFlush = false; flushQueue.clear(); blockedFlushes.clear(); updateStallState(); Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/IndexReaderContext.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/IndexReaderContext.java?rev=1369470&r1=1369469&r2=1369470&view=diff ============================================================================== --- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/IndexReaderContext.java (original) +++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/IndexReaderContext.java Sat Aug 4 21:26:10 2012 @@ -1,7 +1,5 @@ package org.apache.lucene.index; -import java.util.List; - /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with @@ -19,6 +17,8 @@ import java.util.List; * limitations under the License. */ +import java.util.List; + /** * A struct like class that represents a hierarchical relationship between * {@link IndexReader} instances. Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java?rev=1369470&r1=1369469&r2=1369470&view=diff ============================================================================== --- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java (original) +++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java Sat Aug 4 21:26:10 2012 @@ -1241,6 +1241,78 @@ public class IndexWriter implements Clos } } + /** Expert: attempts to delete by document ID, as long as + * the provided reader is a near-real-time reader (from {@link + * DirectoryReader#open(IndexWriter,boolean)}). If the + * provided reader is an NRT reader obtained from this + * writer, and its segment has not been merged away, then + * the delete succeeds and this method returns true; else, it + * returns false the caller must then separately delete by + * Term or Query. + * + * NOTE: this method can only delete documents + * visible to the currently open NRT reader. If you need + * to delete documents indexed after opening the NRT + * reader you must use the other deleteDocument methods + * (e.g., {@link #deleteDocuments(Term)}). */ + public synchronized boolean tryDeleteDocument(IndexReader readerIn, int docID) throws IOException { + + final AtomicReader reader; + if (readerIn instanceof AtomicReader) { + // Reader is already atomic: use the incoming docID: + reader = (AtomicReader) readerIn; + } else { + // Composite reader: lookup sub-reader and re-base docID: + List leaves = readerIn.getTopReaderContext().leaves(); + int subIndex = ReaderUtil.subIndex(docID, leaves); + reader = leaves.get(subIndex).reader(); + docID -= leaves.get(subIndex).docBase; + assert docID >= 0; + assert docID < reader.maxDoc(); + } + + if (!(reader instanceof SegmentReader)) { + throw new IllegalArgumentException("the reader must be a SegmentReader or composite reader containing only SegmentReaders"); + } + + final SegmentInfoPerCommit info = ((SegmentReader) reader).getSegmentInfo(); + + // TODO: this is a slow linear search, but, number of + // segments should be contained unless something is + // seriously wrong w/ the index, so it should be a minor + // cost: + + if (segmentInfos.indexOf(info) != -1) { + ReadersAndLiveDocs rld = readerPool.get(info, false); + if (rld != null) { + synchronized(bufferedDeletesStream) { + rld.initWritableLiveDocs(); + if (rld.delete(docID)) { + final int fullDelCount = rld.info.getDelCount() + rld.getPendingDeleteCount(); + if (fullDelCount == rld.info.info.getDocCount()) { + // If a merge has already registered for this + // segment, we leave it in the readerPool; the + // merge will skip merging it and will then drop + // it once it's done: + if (!mergingSegments.contains(rld.info)) { + segmentInfos.remove(rld.info); + readerPool.drop(rld.info); + checkpoint(); + } + } + } + //System.out.println(" yes " + info.info.name + " " + docID); + return true; + } + } else { + //System.out.println(" no rld " + info.info.name + " " + docID); + } + } else { + //System.out.println(" no seg " + info.info.name + " " + docID); + } + return false; + } + /** * Deletes the document(s) containing any of the * terms. All given deletes are applied and flushed atomically Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/IndexWriterConfig.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/IndexWriterConfig.java?rev=1369470&r1=1369469&r2=1369470&view=diff ============================================================================== --- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/IndexWriterConfig.java (original) +++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/IndexWriterConfig.java Sat Aug 4 21:26:10 2012 @@ -392,7 +392,7 @@ public final class IndexWriterConfig ext * @see #setMaxBufferedDocs(int) * @see #setRAMBufferSizeMB(double) */ - public IndexWriterConfig setFlushPolicy(FlushPolicy flushPolicy) { + IndexWriterConfig setFlushPolicy(FlushPolicy flushPolicy) { this.flushPolicy = flushPolicy; return this; } @@ -422,7 +422,7 @@ public final class IndexWriterConfig ext } @Override - public FlushPolicy getFlushPolicy() { + FlushPolicy getFlushPolicy() { return flushPolicy; } Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/LiveIndexWriterConfig.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/LiveIndexWriterConfig.java?rev=1369470&r1=1369469&r2=1369470&view=diff ============================================================================== --- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/LiveIndexWriterConfig.java (original) +++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/LiveIndexWriterConfig.java Sat Aug 4 21:26:10 2012 @@ -19,6 +19,7 @@ package org.apache.lucene.index; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.codecs.Codec; +import org.apache.lucene.codecs.lucene40.Lucene40PostingsFormat; // javadocs import org.apache.lucene.index.DocumentsWriterPerThread.IndexingChain; import org.apache.lucene.index.IndexWriter.IndexReaderWarmer; import org.apache.lucene.index.IndexWriterConfig.OpenMode; @@ -146,6 +147,29 @@ public class LiveIndexWriterConfig { * Takes effect immediately, but only applies to newly flushed/merged * segments. * + *

+ * NOTE: This parameter does not apply to all PostingsFormat implementations, + * including the default one in this release. It only makes sense for term indexes + * that are implemented as a fixed gap between terms. For example, + * {@link Lucene40PostingsFormat} implements the term index instead based upon how + * terms share prefixes. To configure its parameters (the minimum and maximum size + * for a block), you would instead use {@link Lucene40PostingsFormat#Lucene40PostingsFormat(int, int)}. + * which can also be configured on a per-field basis: + * 

+   * //customize Lucene40PostingsFormat, passing minBlockSize=50, maxBlockSize=100
+   * final PostingsFormat tweakedPostings = new Lucene40PostingsFormat(50, 100);
+   * iwc.setCodec(new Lucene40Codec() {
+   *   @Override
+   *   public PostingsFormat getPostingsFormatForField(String field) {
+   *     if (field.equals("fieldWithTonsOfTerms"))
+   *       return tweakedPostings;
+   *     else
+   *       return super.getPostingsFormatForField(field);
+   *   }
+   * });
+   *
+ * Note that other implementations may have their own parameters, or no parameters at all. + * * @see IndexWriterConfig#DEFAULT_TERM_INDEX_INTERVAL */ public LiveIndexWriterConfig setTermIndexInterval(int interval) { // TODO: this should be private to the codec, not settable here @@ -335,6 +359,10 @@ public class LiveIndexWriterConfig { *

* Takes effect immediately, but only applies to readers opened after this * call + *

+ * NOTE: divisor settings > 1 do not apply to all PostingsFormat + * implementations, including the default one in this release. It only makes + * sense for terms indexes that can efficiently re-sample terms at load time. */ public LiveIndexWriterConfig setReaderTermsIndexDivisor(int divisor) { if (divisor <= 0 && divisor != -1) { @@ -462,7 +490,7 @@ public class LiveIndexWriterConfig { /** * @see IndexWriterConfig#setFlushPolicy(FlushPolicy) */ - public FlushPolicy getFlushPolicy() { + FlushPolicy getFlushPolicy() { return flushPolicy; } @@ -497,7 +525,6 @@ public class LiveIndexWriterConfig { sb.append("mergePolicy=").append(getMergePolicy()).append("\n"); sb.append("indexerThreadPool=").append(getIndexerThreadPool()).append("\n"); sb.append("readerPooling=").append(getReaderPooling()).append("\n"); - sb.append("flushPolicy=").append(getFlushPolicy()).append("\n"); sb.append("perThreadHardLimitMB=").append(getRAMPerThreadHardLimitMB()).append("\n"); return sb.toString(); } Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/FuzzyTermsEnum.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/FuzzyTermsEnum.java?rev=1369470&r1=1369469&r2=1369470&view=diff ============================================================================== --- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/FuzzyTermsEnum.java (original) +++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/FuzzyTermsEnum.java Sat Aug 4 21:26:10 2012 @@ -122,7 +122,7 @@ public class FuzzyTermsEnum extends Term this.realPrefixLength = prefixLength > termLength ? termLength : prefixLength; // if minSimilarity >= 1, we treat it as number of edits if (minSimilarity >= 1f) { - this.minSimilarity = 1 - (minSimilarity+1) / this.termLength; + this.minSimilarity = 0; // just driven by number of edits maxEdits = (int) minSimilarity; raw = true; } else { Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/NRTManager.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/NRTManager.java?rev=1369470&r1=1369469&r2=1369470&view=diff ============================================================================== --- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/NRTManager.java (original) +++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/NRTManager.java Sat Aug 4 21:26:10 2012 @@ -27,6 +27,7 @@ import java.util.concurrent.locks.Reentr import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.SegmentInfoPerCommit; import org.apache.lucene.index.IndexReader; // javadocs import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexableField; @@ -254,6 +255,14 @@ public class NRTManager extends Referenc long getAndIncrementGeneration() { return indexingGen.getAndIncrement(); } + + public long tryDeleteDocument(IndexReader reader, int docID) throws IOException { + if (writer.tryDeleteDocument(reader, docID)) { + return indexingGen.get(); + } else { + return -1; + } + } } /** Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/util/FixedBitSet.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/util/FixedBitSet.java?rev=1369470&r1=1369469&r2=1369470&view=diff ============================================================================== --- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/util/FixedBitSet.java (original) +++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/util/FixedBitSet.java Sat Aug 4 21:26:10 2012 @@ -54,6 +54,11 @@ public final class FixedBitSet extends D bits = new long[bits2words(numBits)]; } + public FixedBitSet(long[]storedBits,int numBits) { + this.numBits = numBits; + this.bits = storedBits; + } + /** Makes full copy. */ public FixedBitSet(FixedBitSet other) { bits = new long[other.bits.length]; Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat?rev=1369470&r1=1369469&r2=1369470&view=diff ============================================================================== --- lucene/dev/branches/pforcodec_3892/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat (original) +++ lucene/dev/branches/pforcodec_3892/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat Sat Aug 4 21:26:10 2012 @@ -18,6 +18,7 @@ org.apache.lucene.codecs.pulsing.Pulsing org.apache.lucene.codecs.simpletext.SimpleTextPostingsFormat org.apache.lucene.codecs.memory.MemoryPostingsFormat org.apache.lucene.codecs.bulkvint.BulkVIntPostingsFormat +org.apache.lucene.codecs.bloom.BloomFilteringPostingsFormat +org.apache.lucene.codecs.memory.DirectPostingsFormat org.apache.lucene.codecs.block.BlockPostingsFormat org.apache.lucene.codecs.blockpacked.BlockPackedPostingsFormat -org.apache.lucene.codecs.memory.DirectPostingsFormat Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/document/TestDocument.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/document/TestDocument.java?rev=1369470&r1=1369469&r2=1369470&view=diff ============================================================================== --- lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/document/TestDocument.java (original) +++ lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/document/TestDocument.java Sat Aug 4 21:26:10 2012 @@ -290,33 +290,4 @@ public class TestDocument extends Lucene // expected } } - - public void testBoost() throws Exception { - Directory dir = newDirectory(); - IndexWriterConfig iwc = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())); - iwc.setMergePolicy(newLogMergePolicy()); - IndexWriter iw = new IndexWriter(dir, iwc); - Document doc = new Document(); - doc.add(new StringField("field1", "sometext", Field.Store.YES)); - doc.add(new TextField("field2", "sometext", Field.Store.NO)); - doc.add(new StringField("foo", "bar", Field.Store.NO)); - iw.addDocument(doc); // add an 'ok' document - try { - doc = new Document(); - // try to boost with norms omitted - StringField field = new StringField("foo", "baz", Field.Store.NO); - field.setBoost(5.0f); - doc.add(field); - iw.addDocument(doc); - fail("didn't get any exception, boost silently discarded"); - } catch (UnsupportedOperationException expected) { - // expected - } - DirectoryReader ir = DirectoryReader.open(iw, false); - assertEquals(1, ir.numDocs()); - assertEquals("sometext", ir.document(0).get("field1")); - ir.close(); - iw.close(); - dir.close(); - } } Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestByteSlices.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestByteSlices.java?rev=1369470&r1=1369469&r2=1369470&view=diff ============================================================================== --- lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestByteSlices.java (original) +++ lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestByteSlices.java Sat Aug 4 21:26:10 2012 @@ -40,24 +40,41 @@ public class TestByteSlices extends Luce counters[stream] = 0; } - int num = atLeast(10000); + int num = atLeast(3000); for (int iter = 0; iter < num; iter++) { - int stream = random().nextInt(NUM_STREAM); - if (VERBOSE) + int stream; + if (random().nextBoolean()) { + stream = random().nextInt(3); + } else { + stream = random().nextInt(NUM_STREAM); + } + + if (VERBOSE) { System.out.println("write stream=" + stream); + } if (starts[stream] == -1) { final int spot = pool.newSlice(ByteBlockPool.FIRST_LEVEL_SIZE); starts[stream] = uptos[stream] = spot + pool.byteOffset; - if (VERBOSE) + if (VERBOSE) { System.out.println(" init to " + starts[stream]); + } } writer.init(uptos[stream]); - int numValue = random().nextInt(20); + int numValue; + if (random().nextInt(10) == 3) { + numValue = random().nextInt(100); + } else if (random().nextInt(5) == 3) { + numValue = random().nextInt(3); + } else { + numValue = random().nextInt(20); + } + for(int j=0;j list = new ArrayList(); + list.add(new IndexableField() { + + @Override + public String name() { + return "foo"; + } + + @Override + public IndexableFieldType fieldType() { + return StringField.TYPE_NOT_STORED; + } + + @Override + public float boost() { + return 5f; + } + + @Override + public BytesRef binaryValue() { + return null; + } + + @Override + public String stringValue() { + return "baz"; + } + + @Override + public Reader readerValue() { + return null; + } + + @Override + public Number numericValue() { + return null; + } + + @Override + public TokenStream tokenStream(Analyzer analyzer) throws IOException { + return null; + } + }); + iw.addDocument(list); + fail("didn't get any exception, boost silently discarded"); + } catch (UnsupportedOperationException expected) { + // expected + } + DirectoryReader ir = DirectoryReader.open(iw, false); + assertEquals(1, ir.numDocs()); + assertEquals("sometext", ir.document(0).get("field1")); + ir.close(); + iw.close(); + dir.close(); + } } Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestRollingUpdates.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestRollingUpdates.java?rev=1369470&r1=1369469&r2=1369470&view=diff ============================================================================== --- lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestRollingUpdates.java (original) +++ lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestRollingUpdates.java Sat Aug 4 21:26:10 2012 @@ -23,6 +23,9 @@ import org.apache.lucene.analysis.MockAn import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.memory.MemoryPostingsFormat; import org.apache.lucene.document.*; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.search.TopDocs; import org.apache.lucene.store.*; import org.apache.lucene.util.*; import org.junit.Test; @@ -48,10 +51,13 @@ public class TestRollingUpdates extends final int SIZE = atLeast(20); int id = 0; IndexReader r = null; + IndexSearcher s = null; final int numUpdates = (int) (SIZE * (2+(TEST_NIGHTLY ? 200*random().nextDouble() : 5*random().nextDouble()))); if (VERBOSE) { System.out.println("TEST: numUpdates=" + numUpdates); } + int updateCount = 0; + // TODO: sometimes update ids not in order... for(int docIter=0;docIter= SIZE && random().nextInt(50) == 17) { if (r != null) { r.close(); } + final boolean applyDeletions = random().nextBoolean(); + + if (VERBOSE) { + System.out.println("TEST: reopen applyDeletions=" + applyDeletions); + } + r = w.getReader(applyDeletions); + if (applyDeletions) { + s = new IndexSearcher(r); + } else { + s = null; + } assertTrue("applyDeletions=" + applyDeletions + " r.numDocs()=" + r.numDocs() + " vs SIZE=" + SIZE, !applyDeletions || r.numDocs() == SIZE); + updateCount = 0; } } Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/search/TestFuzzyQuery.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/search/TestFuzzyQuery.java?rev=1369470&r1=1369469&r2=1369470&view=diff ============================================================================== --- lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/search/TestFuzzyQuery.java (original) +++ lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/search/TestFuzzyQuery.java Sat Aug 4 21:26:10 2012 @@ -22,6 +22,7 @@ import java.util.Arrays; import java.io.IOException; import org.apache.lucene.analysis.MockAnalyzer; +import org.apache.lucene.analysis.MockTokenizer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.index.IndexReader; @@ -189,6 +190,41 @@ public class TestFuzzyQuery extends Luce directory.close(); } + public void test2() throws Exception { + Directory directory = newDirectory(); + RandomIndexWriter writer = new RandomIndexWriter(random(), directory, new MockAnalyzer(random(), MockTokenizer.KEYWORD, false)); + addDoc("LANGE", writer); + addDoc("LUETH", writer); + addDoc("PIRSING", writer); + addDoc("RIEGEL", writer); + addDoc("TRZECZIAK", writer); + addDoc("WALKER", writer); + addDoc("WBR", writer); + addDoc("WE", writer); + addDoc("WEB", writer); + addDoc("WEBE", writer); + addDoc("WEBER", writer); + addDoc("WEBERE", writer); + addDoc("WEBREE", writer); + addDoc("WEBEREI", writer); + addDoc("WBRE", writer); + addDoc("WITTKOPF", writer); + addDoc("WOJNAROWSKI", writer); + addDoc("WRICKE", writer); + + IndexReader reader = writer.getReader(); + IndexSearcher searcher = newSearcher(reader); + writer.close(); + + FuzzyQuery query = new FuzzyQuery(new Term("field", "WEBER"), 2, 1); + //query.setRewriteMethod(FuzzyQuery.SCORING_BOOLEAN_QUERY_REWRITE); + ScoreDoc[] hits = searcher.search(query, null, 1000).scoreDocs; + assertEquals(8, hits.length); + + reader.close(); + directory.close(); + } + /** * MultiTermQuery provides (via attribute) information about which values * must be competitive to enter the priority queue. Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/search/TestPositionIncrement.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/search/TestPositionIncrement.java?rev=1369470&r1=1369469&r2=1369470&view=diff ============================================================================== --- lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/search/TestPositionIncrement.java (original) +++ lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/search/TestPositionIncrement.java Sat Aug 4 21:26:10 2012 @@ -206,7 +206,7 @@ public class TestPositionIncrement exten RandomIndexWriter writer = new RandomIndexWriter(random(), dir, new MockPayloadAnalyzer()); Document doc = new Document(); doc.add(new TextField("content", new StringReader( - "a a b c d e a f g h i j a b k k"), Field.Store.NO)); + "a a b c d e a f g h i j a b k k"))); writer.addDocument(doc); final IndexReader readerFromWriter = writer.getReader(); Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/search/TestShardSearching.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/search/TestShardSearching.java?rev=1369470&r1=1369469&r2=1369470&view=diff ============================================================================== --- lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/search/TestShardSearching.java (original) +++ lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/search/TestShardSearching.java Sat Aug 4 21:26:10 2012 @@ -77,8 +77,7 @@ public class TestShardSearching extends System.out.println("TEST: numNodes=" + numNodes + " runTimeSec=" + runTimeSec + " maxSearcherAgeSeconds=" + maxSearcherAgeSeconds); } - start(_TestUtil.getTempDir("TestShardSearching").toString(), - numNodes, + start(numNodes, runTimeSec, maxSearcherAgeSeconds ); Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/search/TestSort.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/search/TestSort.java?rev=1369470&r1=1369469&r2=1369470&view=diff ============================================================================== --- lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/search/TestSort.java (original) +++ lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/search/TestSort.java Sat Aug 4 21:26:10 2012 @@ -196,7 +196,7 @@ public class TestSort extends LuceneTest if (data[i][11] != null) doc.add (new StringField ("parser", data[i][11], Field.Store.NO)); for(IndexableField f : doc.getFields()) { - if (!f.fieldType().omitNorms()) { + if (f.fieldType().indexed() && !f.fieldType().omitNorms()) { ((Field) f).setBoost(2.0f); } } @@ -239,7 +239,7 @@ public class TestSort extends LuceneTest doc.add(new SortedBytesDocValuesField("string2", new BytesRef(num2))); doc.add (new Field ("tracer2", num2, onlyStored)); for(IndexableField f2 : doc.getFields()) { - if (!f2.fieldType().omitNorms()) { + if (f2.fieldType().indexed() && !f2.fieldType().omitNorms()) { ((Field) f2).setBoost(2.0f); } } @@ -255,7 +255,7 @@ public class TestSort extends LuceneTest doc.add (new Field ("tracer2_fixed", num2Fixed, onlyStored)); for(IndexableField f2 : doc.getFields()) { - if (!f2.fieldType().omitNorms()) { + if (f2.fieldType().indexed() && !f2.fieldType().omitNorms()) { ((Field) f2).setBoost(2.0f); } } Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/search/spans/TestPayloadSpans.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/search/spans/TestPayloadSpans.java?rev=1369470&r1=1369469&r2=1369470&view=diff ============================================================================== --- lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/search/spans/TestPayloadSpans.java (original) +++ lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/search/spans/TestPayloadSpans.java Sat Aug 4 21:26:10 2012 @@ -256,7 +256,7 @@ public class TestPayloadSpans extends Lu newIndexWriterConfig(TEST_VERSION_CURRENT, new TestPayloadAnalyzer())); Document doc = new Document(); - doc.add(new TextField("content", new StringReader("a b c d e f g h i j a k"), Field.Store.NO)); + doc.add(new TextField("content", new StringReader("a b c d e f g h i j a k"))); writer.addDocument(doc); IndexReader reader = writer.getReader(); @@ -293,7 +293,7 @@ public class TestPayloadSpans extends Lu newIndexWriterConfig(TEST_VERSION_CURRENT, new TestPayloadAnalyzer())); Document doc = new Document(); - doc.add(new TextField("content", new StringReader("a b a d k f a h i k a k"), Field.Store.NO)); + doc.add(new TextField("content", new StringReader("a b a d k f a h i k a k"))); writer.addDocument(doc); IndexReader reader = writer.getReader(); IndexSearcher is = newSearcher(reader); @@ -328,7 +328,7 @@ public class TestPayloadSpans extends Lu newIndexWriterConfig(TEST_VERSION_CURRENT, new TestPayloadAnalyzer())); Document doc = new Document(); - doc.add(new TextField("content", new StringReader("j k a l f k k p a t a k l k t a"), Field.Store.NO)); + doc.add(new TextField("content", new StringReader("j k a l f k k p a t a k l k t a"))); writer.addDocument(doc); IndexReader reader = writer.getReader(); IndexSearcher is = newSearcher(reader); Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/util/fst/TestFSTs.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/util/fst/TestFSTs.java?rev=1369470&r1=1369469&r2=1369470&view=diff ============================================================================== --- lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/util/fst/TestFSTs.java (original) +++ lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/util/fst/TestFSTs.java Sat Aug 4 21:26:10 2012 @@ -1096,13 +1096,6 @@ public class TestFSTs extends LuceneTest // file, up until a time limit public void testRealTerms() throws Exception { - // TODO: is this necessary? we use the annotation... - final String defaultFormat = _TestUtil.getPostingsFormat("abracadabra"); - if (defaultFormat.equals("SimpleText") || defaultFormat.equals("Memory")) { - // no - Codec.setDefault(_TestUtil.alwaysPostingsFormat(new Lucene40PostingsFormat())); - } - final LineFileDocs docs = new LineFileDocs(random(), true); final int RUN_TIME_MSEC = atLeast(500); final IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())).setMaxBufferedDocs(-1).setRAMBufferSizeMB(64); Modified: lucene/dev/branches/pforcodec_3892/lucene/demo/src/java/org/apache/lucene/demo/IndexFiles.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/demo/src/java/org/apache/lucene/demo/IndexFiles.java?rev=1369470&r1=1369469&r2=1369470&view=diff ============================================================================== --- lucene/dev/branches/pforcodec_3892/lucene/demo/src/java/org/apache/lucene/demo/IndexFiles.java (original) +++ lucene/dev/branches/pforcodec_3892/lucene/demo/src/java/org/apache/lucene/demo/IndexFiles.java Sat Aug 4 21:26:10 2012 @@ -190,7 +190,7 @@ public class IndexFiles { // so that the text of the file is tokenized and indexed, but not stored. // Note that FileReader expects the file to be in UTF-8 encoding. // If that's not the case searching for special characters will fail. - doc.add(new TextField("contents", new BufferedReader(new InputStreamReader(fis, "UTF-8")), Field.Store.NO)); + doc.add(new TextField("contents", new BufferedReader(new InputStreamReader(fis, "UTF-8")))); if (writer.getConfig().getOpenMode() == OpenMode.CREATE) { // New index, so we just add the document (no old document can be there): Modified: lucene/dev/branches/pforcodec_3892/lucene/sandbox/src/test/org/apache/lucene/sandbox/queries/TestSlowFuzzyQuery.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/sandbox/src/test/org/apache/lucene/sandbox/queries/TestSlowFuzzyQuery.java?rev=1369470&r1=1369469&r2=1369470&view=diff ============================================================================== --- lucene/dev/branches/pforcodec_3892/lucene/sandbox/src/test/org/apache/lucene/sandbox/queries/TestSlowFuzzyQuery.java (original) +++ lucene/dev/branches/pforcodec_3892/lucene/sandbox/src/test/org/apache/lucene/sandbox/queries/TestSlowFuzzyQuery.java Sat Aug 4 21:26:10 2012 @@ -440,21 +440,25 @@ public class TestSlowFuzzyQuery extends assertEquals(1, hits.length); assertEquals("foobar", searcher.doc(hits[0].doc).get("field")); - q = new SlowFuzzyQuery(new Term("field", "t"), 3); - hits = searcher.search(q, 10).scoreDocs; - assertEquals(1, hits.length); - assertEquals("test", searcher.doc(hits[0].doc).get("field")); + // TODO: cannot really be supported given the legacy scoring + // system which scores negative, if the distance > min term len, + // so such matches were always impossible with lucene 3.x, etc + // + //q = new SlowFuzzyQuery(new Term("field", "t"), 3); + //hits = searcher.search(q, 10).scoreDocs; + //assertEquals(1, hits.length); + //assertEquals("test", searcher.doc(hits[0].doc).get("field")); - q = new SlowFuzzyQuery(new Term("field", "a"), 4f, 0, 50); - hits = searcher.search(q, 10).scoreDocs; - assertEquals(1, hits.length); - assertEquals("test", searcher.doc(hits[0].doc).get("field")); + // q = new SlowFuzzyQuery(new Term("field", "a"), 4f, 0, 50); + // hits = searcher.search(q, 10).scoreDocs; + // assertEquals(1, hits.length); + // assertEquals("test", searcher.doc(hits[0].doc).get("field")); - q = new SlowFuzzyQuery(new Term("field", "a"), 6f, 0, 50); - hits = searcher.search(q, 10).scoreDocs; - assertEquals(2, hits.length); - assertEquals("test", searcher.doc(hits[0].doc).get("field")); - assertEquals("foobar", searcher.doc(hits[1].doc).get("field")); + // q = new SlowFuzzyQuery(new Term("field", "a"), 6f, 0, 50); + // hits = searcher.search(q, 10).scoreDocs; + // assertEquals(2, hits.length); + // assertEquals("test", searcher.doc(hits[0].doc).get("field")); + // assertEquals("foobar", searcher.doc(hits[1].doc).get("field")); reader.close(); index.close(); Modified: lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/index/RandomCodec.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/index/RandomCodec.java?rev=1369470&r1=1369469&r2=1369470&view=diff ============================================================================== --- lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/index/RandomCodec.java (original) +++ lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/index/RandomCodec.java Sat Aug 4 21:26:10 2012 @@ -29,6 +29,7 @@ import java.util.Set; import org.apache.lucene.codecs.PostingsFormat; import org.apache.lucene.codecs.asserting.AssertingPostingsFormat; +import org.apache.lucene.codecs.bloom.TestBloomFilteredLucene40Postings; import org.apache.lucene.codecs.lucene40.Lucene40Codec; import org.apache.lucene.codecs.lucene40.Lucene40PostingsFormat; import org.apache.lucene.codecs.lucene40ords.Lucene40WithOrds; @@ -98,6 +99,10 @@ public class RandomCodec extends Lucene4 new Pulsing40PostingsFormat(1 + random.nextInt(20), minItemsPerBlock, maxItemsPerBlock), // add pulsing again with (usually) different parameters new Pulsing40PostingsFormat(1 + random.nextInt(20), minItemsPerBlock, maxItemsPerBlock), + //TODO as a PostingsFormat which wraps others, we should allow TestBloomFilteredLucene40Postings to be constructed + //with a choice of concrete PostingsFormats. Maybe useful to have a generic means of marking and dealing + //with such "wrapper" classes? + new TestBloomFilteredLucene40Postings(), new MockSepPostingsFormat(), new MockFixedIntBlockPostingsFormat(_TestUtil.nextInt(random, 1, 2000)), new MockVariableIntBlockPostingsFormat( _TestUtil.nextInt(random, 1, 127)), Modified: lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/search/ShardSearchingTestBase.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/search/ShardSearchingTestBase.java?rev=1369470&r1=1369469&r2=1369470&view=diff ============================================================================== --- lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/search/ShardSearchingTestBase.java (original) +++ lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/search/ShardSearchingTestBase.java Sat Aug 4 21:26:10 2012 @@ -18,7 +18,6 @@ package org.apache.lucene.search; */ import java.io.Closeable; -import java.io.File; import java.io.IOException; import java.util.HashMap; import java.util.HashSet; @@ -27,15 +26,17 @@ import java.util.Random; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.Term; import org.apache.lucene.index.TermContext; -import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.store.Directory; import org.apache.lucene.util.LineFileDocs; import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util.PrintStreamInfoStream; +import org.apache.lucene.util._TestUtil; // TODO // - doc blocks? so we can test joins/grouping... @@ -423,11 +424,16 @@ public abstract class ShardSearchingTest private volatile ShardIndexSearcher currentShardSearcher; - public NodeState(Random random, String baseDir, int nodeID, int numNodes) throws IOException { + public NodeState(Random random, int nodeID, int numNodes) throws IOException { myNodeID = nodeID; - dir = newFSDirectory(new File(baseDir + "." + myNodeID)); + dir = newFSDirectory(_TestUtil.getTempDir("ShardSearchingTestBase")); // TODO: set warmer - writer = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))); + IndexWriterConfig iwc = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)); + iwc.setOpenMode(IndexWriterConfig.OpenMode.CREATE); + if (VERBOSE) { + iwc.setInfoStream(new PrintStreamInfoStream(System.out)); + } + writer = new IndexWriter(dir, iwc); mgr = new SearcherManager(writer, true, null); searchers = new SearcherLifetimeManager(); @@ -556,14 +562,14 @@ public abstract class ShardSearchingTest long endTimeNanos; private Thread changeIndicesThread; - protected void start(String baseDirName, int numNodes, double runTimeSec, int maxSearcherAgeSeconds) throws IOException { + protected void start(int numNodes, double runTimeSec, int maxSearcherAgeSeconds) throws IOException { endTimeNanos = System.nanoTime() + (long) (runTimeSec*1000000000); this.maxSearcherAgeSeconds = maxSearcherAgeSeconds; nodes = new NodeState[numNodes]; for(int nodeID=0;nodeID 0) { throw new RuntimeException("MockDirectoryWrapper: cannot close: there are still open locks: " + openLocks); } - open = false; + isOpen = false; if (getCheckIndexOnClose()) { if (indexPossiblyExists()) { if (LuceneTestCase.VERBOSE) { @@ -614,11 +614,6 @@ public class MockDirectoryWrapper extend public synchronized void removeIndexInput(IndexInput in, String name) { removeOpenFile(in, name); } - - @Override - public synchronized boolean isOpen() { - return open; - } /** * Objects that represent fail-able conditions. Objects of a derived Modified: lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat?rev=1369470&r1=1369469&r2=1369470&view=diff ============================================================================== --- lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat (original) +++ lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat Sat Aug 4 21:26:10 2012 @@ -20,5 +20,6 @@ org.apache.lucene.codecs.mocksep.MockSep org.apache.lucene.codecs.nestedpulsing.NestedPulsingPostingsFormat org.apache.lucene.codecs.ramonly.RAMOnlyPostingsFormat org.apache.lucene.codecs.lucene40ords.Lucene40WithOrds +org.apache.lucene.codecs.bloom.TestBloomFilteredLucene40Postings org.apache.lucene.codecs.asserting.AssertingPostingsFormat