Return-Path: X-Original-To: apmail-lucene-commits-archive@www.apache.org Delivered-To: apmail-lucene-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 0AC279052 for ; Sun, 10 Jun 2012 14:43:00 +0000 (UTC) Received: (qmail 69531 invoked by uid 500); 10 Jun 2012 14:42:59 -0000 Mailing-List: contact commits-help@lucene.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@lucene.apache.org Delivered-To: mailing list commits@lucene.apache.org Received: (qmail 69521 invoked by uid 99); 10 Jun 2012 14:42:59 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Sun, 10 Jun 2012 14:42:59 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Sun, 10 Jun 2012 14:42:56 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 1314723889BF; Sun, 10 Jun 2012 14:42:35 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1348606 - in /lucene/dev/trunk/lucene: ./ core/src/java/org/apache/lucene/index/ core/src/test/org/apache/lucene/index/ core/src/test/org/apache/lucene/search/ facet/src/java/org/apache/lucene/facet/taxonomy/directory/ test-framework/src/j... Date: Sun, 10 Jun 2012 14:42:34 -0000 To: commits@lucene.apache.org From: mikemccand@apache.org X-Mailer: svnmailer-1.0.8-patched Message-Id: <20120610144235.1314723889BF@eris.apache.org> Author: mikemccand Date: Sun Jun 10 14:42:34 2012 New Revision: 1348606 URL: http://svn.apache.org/viewvc?rev=1348606&view=rev Log: LUCENE-4127: don't allow 0 posInc for first token of indexed field Modified: lucene/dev/trunk/lucene/CHANGES.txt lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocInverterPerField.java lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/TestPositionIncrement.java lucene/dev/trunk/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/directory/DirectoryTaxonomyWriter.java lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/analysis/MockPayloadAnalyzer.java Modified: lucene/dev/trunk/lucene/CHANGES.txt URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/CHANGES.txt?rev=1348606&r1=1348605&r2=1348606&view=diff ============================================================================== --- lucene/dev/trunk/lucene/CHANGES.txt (original) +++ lucene/dev/trunk/lucene/CHANGES.txt Sun Jun 10 14:42:34 2012 @@ -393,6 +393,11 @@ Changes in Runtime Behavior any of the calls to the Analyzer throw an IOException. QueryParseBase.analyzeRangePart() will throw a RuntimException if an IOException is thrown by the Analyzer. +* LUCENE-4127: IndexWriter will now throw IllegalArgumentException if + the first token of an indexed field has 0 positionIncrement + (previously it silently corrected it to 1, possibly masking bugs). + (Robert Muir, Mike McCandless) + API Changes * LUCENE-2302, LUCENE-1458, LUCENE-2111, LUCENE-2514: Terms are no longer Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocInverterPerField.java URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocInverterPerField.java?rev=1348606&r1=1348605&r2=1348606&view=diff ============================================================================== --- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocInverterPerField.java (original) +++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocInverterPerField.java Sun Jun 10 14:42:34 2012 @@ -18,9 +18,11 @@ package org.apache.lucene.index; */ import java.io.IOException; + import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.tokenattributes.OffsetAttribute; import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; +import org.apache.lucene.util.IOUtils; /** * Holds state for inverting all occurrences of a single @@ -87,6 +89,8 @@ final class DocInverterPerField extends // reset the TokenStream to the first token stream.reset(); + boolean success2 = false; + try { boolean hasMoreTokens = stream.incrementToken(); @@ -109,8 +113,16 @@ final class DocInverterPerField extends if (!hasMoreTokens) break; final int posIncr = posIncrAttribute.getPositionIncrement(); + if (posIncr < 0) { + throw new IllegalArgumentException("position increment must be >=0 (got " + posIncr + ")"); + } + if (fieldState.position == 0 && posIncr == 0) { + throw new IllegalArgumentException("first position increment must be > 0 (got 0)"); + } int position = fieldState.position + posIncr; if (position > 0) { + // NOTE: confusing: this "mirrors" the + // position++ we do below position--; } else if (position < 0) { throw new IllegalArgumentException("position overflow for field '" + field.name() + "'"); @@ -147,8 +159,13 @@ final class DocInverterPerField extends stream.end(); fieldState.offset += offsetAttribute.endOffset(); + success2 = true; } finally { - stream.close(); + if (!success2) { + IOUtils.closeWhileHandlingException(stream); + } else { + stream.close(); + } } fieldState.offset += docState.analyzer == null ? 0 : docState.analyzer.getOffsetGap(field); Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java?rev=1348606&r1=1348605&r2=1348606&view=diff ============================================================================== --- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java (original) +++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java Sun Jun 10 14:42:34 2012 @@ -883,39 +883,16 @@ public class TestIndexWriter extends Luc IndexWriter w = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random()))); Document doc = new Document(); doc.add(new TextField("field", tokens)); - w.addDocument(doc); - w.commit(); - - IndexReader r = DirectoryReader.open(dir); - IndexSearcher s = new IndexSearcher(r); - PhraseQuery pq = new PhraseQuery(); - pq.add(new Term("field", "a")); - pq.add(new Term("field", "b")); - pq.add(new Term("field", "c")); - ScoreDoc[] hits = s.search(pq, null, 1000).scoreDocs; - assertEquals(1, hits.length); - - Query q = new SpanTermQuery(new Term("field", "a")); - hits = s.search(q, null, 1000).scoreDocs; - assertEquals(1, hits.length); - - DocsAndPositionsEnum tps = MultiFields.getTermPositionsEnum(s.getIndexReader(), - MultiFields.getLiveDocs(s.getIndexReader()), - "field", - new BytesRef("a"), - false); - - assertTrue(tps.nextDoc() != DocIdSetIterator.NO_MORE_DOCS); - assertEquals(1, tps.freq()); - assertEquals(0, tps.nextPosition()); + try { + w.addDocument(doc); + fail("did not hit expected exception"); + } catch (IllegalArgumentException iea) { + // expected + } w.close(); - - r.close(); dir.close(); } - - // LUCENE-1219 public void testBinaryFieldOffsetLength() throws IOException { Directory dir = newDirectory(); Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/TestPositionIncrement.java URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/TestPositionIncrement.java?rev=1348606&r1=1348605&r2=1348606&view=diff ============================================================================== --- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/TestPositionIncrement.java (original) +++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/TestPositionIncrement.java Sun Jun 10 14:42:34 2012 @@ -61,7 +61,7 @@ public class TestPositionIncrement exten return new TokenStreamComponents(new Tokenizer(reader) { // TODO: use CannedTokenStream private final String[] TOKENS = {"1", "2", "3", "4", "5"}; - private final int[] INCREMENTS = {0, 2, 1, 0, 1}; + private final int[] INCREMENTS = {1, 2, 1, 0, 1}; private int i = 0; PositionIncrementAttribute posIncrAtt = addAttribute(PositionIncrementAttribute.class); @@ -222,8 +222,7 @@ public class TestPositionIncrement exten assertTrue(tp.nextDoc() != DocIdSetIterator.NO_MORE_DOCS); // "a" occurs 4 times assertEquals(4, tp.freq()); - int expected = 0; - assertEquals(expected, tp.nextPosition()); + assertEquals(0, tp.nextPosition()); assertEquals(1, tp.nextPosition()); assertEquals(3, tp.nextPosition()); assertEquals(6, tp.nextPosition()); Modified: lucene/dev/trunk/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/directory/DirectoryTaxonomyWriter.java URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/directory/DirectoryTaxonomyWriter.java?rev=1348606&r1=1348605&r2=1348606&view=diff ============================================================================== --- lucene/dev/trunk/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/directory/DirectoryTaxonomyWriter.java (original) +++ lucene/dev/trunk/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/directory/DirectoryTaxonomyWriter.java Sun Jun 10 14:42:34 2012 @@ -546,7 +546,7 @@ public class DirectoryTaxonomyWriter imp // we write here (e.g., to write parent+2), and need to do a workaround // in the reader (which knows that anyway only category 0 has a parent // -1). - parentStream.set(parent + 1); + parentStream.set(Math.max(parent+1, 1)); Document d = new Document(); d.add(parentStreamField); Modified: lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/analysis/MockPayloadAnalyzer.java URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/analysis/MockPayloadAnalyzer.java?rev=1348606&r1=1348605&r2=1348606&view=diff ============================================================================== --- lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/analysis/MockPayloadAnalyzer.java (original) +++ lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/analysis/MockPayloadAnalyzer.java Sun Jun 10 14:42:34 2012 @@ -69,7 +69,7 @@ final class MockPayloadFilter extends To if (input.incrementToken()) { payloadAttr.setPayload(new BytesRef(("pos: " + pos).getBytes())); int posIncr; - if (i % 2 == 1) { + if (pos == 0 || i % 2 == 1) { posIncr = 1; } else { posIncr = 0;