Return-Path: Delivered-To: apmail-lucene-java-commits-archive@www.apache.org Received: (qmail 69208 invoked from network); 27 Mar 2010 21:21:58 -0000 Received: from unknown (HELO mail.apache.org) (140.211.11.3) by 140.211.11.9 with SMTP; 27 Mar 2010 21:21:58 -0000 Received: (qmail 50451 invoked by uid 500); 27 Mar 2010 21:21:58 -0000 Delivered-To: apmail-lucene-java-commits-archive@lucene.apache.org Received: (qmail 50425 invoked by uid 500); 27 Mar 2010 21:21:58 -0000 Mailing-List: contact java-commits-help@lucene.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: java-dev@lucene.apache.org Delivered-To: mailing list java-commits@lucene.apache.org Received: (qmail 50415 invoked by uid 99); 27 Mar 2010 21:21:58 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 27 Mar 2010 21:21:58 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=10.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 27 Mar 2010 21:21:50 +0000 Received: by eris.apache.org (Postfix, from userid 65534) id 0BBF92388980; Sat, 27 Mar 2010 21:21:28 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r928283 - in /lucene/java/branches/flex_1458: ./ backwards/src/ backwards/src/java/org/apache/lucene/search/ backwards/src/test/org/apache/lucene/analysis/ backwards/src/test/org/apache/lucene/document/ backwards/src/test/org/apache/lucene/... Date: Sat, 27 Mar 2010 21:21:27 -0000 To: java-commits@lucene.apache.org From: mikemccand@apache.org X-Mailer: svnmailer-1.0.8 Message-Id: <20100327212128.0BBF92388980@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: mikemccand Date: Sat Mar 27 21:21:26 2010 New Revision: 928283 URL: http://svn.apache.org/viewvc?rev=928283&view=rev Log: LUCENE-2329: merge to flex branch (from trunk rev 926791) Added: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/ParallelPostingsArray.java - copied unchanged from r926791, lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/ParallelPostingsArray.java Removed: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/RawPostingList.java Modified: lucene/java/branches/flex_1458/ (props changed) lucene/java/branches/flex_1458/CHANGES.txt lucene/java/branches/flex_1458/backwards/src/ (props changed) lucene/java/branches/flex_1458/backwards/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java (props changed) lucene/java/branches/flex_1458/backwards/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java (props changed) lucene/java/branches/flex_1458/backwards/src/test/org/apache/lucene/document/TestDateTools.java (props changed) lucene/java/branches/flex_1458/backwards/src/test/org/apache/lucene/document/TestNumberTools.java (props changed) lucene/java/branches/flex_1458/backwards/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java (props changed) lucene/java/branches/flex_1458/backwards/src/test/org/apache/lucene/util/TestAttributeSource.java (props changed) lucene/java/branches/flex_1458/build.xml (props changed) lucene/java/branches/flex_1458/contrib/ (props changed) lucene/java/branches/flex_1458/contrib/CHANGES.txt (props changed) lucene/java/branches/flex_1458/contrib/highlighter/src/test/ (props changed) lucene/java/branches/flex_1458/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java (props changed) lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/Tokenizer.java (props changed) lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/FreqProxFieldMergeState.java lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/FreqProxTermsWriter.java lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/FreqProxTermsWriterPerField.java lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/TermVectorsTermsWriter.java lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/TermVectorsTermsWriterPerField.java lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/TermsHash.java lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/TermsHashConsumer.java lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/TermsHashConsumerPerField.java lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/TermsHashPerField.java lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/TermsHashPerThread.java lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java (props changed) lucene/java/branches/flex_1458/src/java/org/apache/lucene/store/Directory.java lucene/java/branches/flex_1458/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java (props changed) lucene/java/branches/flex_1458/src/test/org/apache/lucene/document/TestDateTools.java (props changed) lucene/java/branches/flex_1458/src/test/org/apache/lucene/document/TestNumberTools.java (props changed) lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java (props changed) lucene/java/branches/flex_1458/src/test/org/apache/lucene/util/TestAttributeSource.java (props changed) Propchange: lucene/java/branches/flex_1458/ ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Sat Mar 27 21:21:26 2010 @@ -1,4 +1,4 @@ -/lucene/dev/trunk/lucene:926653,926948-928243,928255 +/lucene/dev/trunk/lucene:926653,926791,926948-928243,928255 /lucene/java/branches/lucene_2_4:748824 /lucene/java/branches/lucene_2_9:817269-818600,829134,829881,831036,896850,909334 /lucene/java/branches/lucene_2_9_back_compat_tests:818601-821336 Modified: lucene/java/branches/flex_1458/CHANGES.txt URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/CHANGES.txt?rev=928283&r1=928282&r2=928283&view=diff ============================================================================== --- lucene/java/branches/flex_1458/CHANGES.txt (original) +++ lucene/java/branches/flex_1458/CHANGES.txt Sat Mar 27 21:21:26 2010 @@ -6,9 +6,11 @@ Changes in backwards compatibility polic * LUCENE-1458, LUCENE-2111: Changes from flexible indexing: - MultiReader ctor now throws IOException - - Directory.copy now copies all files (not just index files), since - what is and isn't and index file is now dependent on the codec - used. (Mike McCandless) + + - Directory.copy/Directory.copyTo now copies all files (not just + index files), since what is and isn't and index file is now + dependent on the codecs used. (Mike McCandless) + - UnicodeUtil now uses BytesRef for UTF-8 output, and some method signatures have changed to CharSequence. These are advanced APIs and subject to change suddenly. (Robert Muir, Mike McCandless) @@ -319,6 +321,15 @@ Optimizations TermAttributeImpl, move DEFAULT_TYPE constant to TypeInterface, improve null-handling for TypeAttribute. (Uwe Schindler) +* LUCENE-2329: Switch TermsHash* from using a PostingList object per unique + term to parallel arrays, indexed by termID. This reduces garbage collection + overhead significantly, which results in great indexing performance wins + when the available JVM heap space is low. This will become even more + important when the DocumentsWriter RAM buffer is searchable in the future, + because then it will make sense to make the RAM buffers as large as + possible. (Mike McCandless, Michael Busch) + + Build * LUCENE-2124: Moved the JDK-based collation support from contrib/collation Propchange: lucene/java/branches/flex_1458/backwards/src/ ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Sat Mar 27 21:21:26 2010 @@ -1,4 +1,4 @@ -/lucene/dev/trunk/lucene/backwards/src:926653,926948-928243,928255 +/lucene/dev/trunk/lucene/backwards/src:926653,926791,926948-928243,928255 /lucene/java/branches/lucene_2_4/src:748824 /lucene/java/branches/lucene_2_9/src:817269-818600,825998,829134,829881,831036 /lucene/java/branches/lucene_2_9_back_compat_tests/src:818601-821336 Propchange: lucene/java/branches/flex_1458/backwards/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Sat Mar 27 21:21:26 2010 @@ -1,4 +1,4 @@ -/lucene/dev/trunk/lucene/backwards/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java:926653,926948-928243,928255 +/lucene/dev/trunk/lucene/backwards/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java:926653,926791,926948-928243,928255 /lucene/java/branches/lucene_2_4/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java:748824 /lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java:817269-818600,825998,829134,829881,831036 /lucene/java/branches/lucene_2_9_back_compat_tests/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java:818601-821336 Propchange: lucene/java/branches/flex_1458/backwards/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Sat Mar 27 21:21:26 2010 @@ -1,4 +1,4 @@ -/lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java:926653,926948-928243,928255 +/lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java:926653,926791,926948-928243,928255 /lucene/java/branches/lucene_2_4/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java:748824 /lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java:825998,829134,829881,831036 /lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java:880754,880793,880823,881216,881317,881376,881473,881549,881820,882467,882890,883076,883080,912383 Propchange: lucene/java/branches/flex_1458/backwards/src/test/org/apache/lucene/document/TestDateTools.java ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Sat Mar 27 21:21:26 2010 @@ -1,4 +1,4 @@ -/lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/document/TestDateTools.java:926653,926948-928243,928255 +/lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/document/TestDateTools.java:926653,926791,926948-928243,928255 /lucene/java/branches/lucene_2_4/src/test/org/apache/lucene/document/TestDateTools.java:748824 /lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/document/TestDateTools.java:825998,829134,829881,831036 /lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/document/TestDateTools.java:880754,880793,880823,881216,881317,881376,881473,881549,881820,882467,882890,883076,883080,912383 Propchange: lucene/java/branches/flex_1458/backwards/src/test/org/apache/lucene/document/TestNumberTools.java ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Sat Mar 27 21:21:26 2010 @@ -1,4 +1,4 @@ -/lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/document/TestNumberTools.java:926653,926948-928243,928255 +/lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/document/TestNumberTools.java:926653,926791,926948-928243,928255 /lucene/java/branches/lucene_2_4/src/test/org/apache/lucene/document/TestNumberTools.java:748824 /lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/document/TestNumberTools.java:825998,829134,829881,831036 /lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/document/TestNumberTools.java:880754,880793,880823,881216,881317,881376,881473,881549,881820,882467,882890,883076,883080,912383 Propchange: lucene/java/branches/flex_1458/backwards/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Sat Mar 27 21:21:26 2010 @@ -1,4 +1,4 @@ -/lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:926653,926948-928243,928255 +/lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:926653,926791,926948-928243,928255 /lucene/java/branches/lucene_2_4/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:748824 /lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:825998,829134,829881,831036 /lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:880754,880793,880823,881216,881317,881376,881473,881549,881820,882467,882890,883076,883080,912383 Propchange: lucene/java/branches/flex_1458/backwards/src/test/org/apache/lucene/util/TestAttributeSource.java ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Sat Mar 27 21:21:26 2010 @@ -1,4 +1,4 @@ -/lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/util/TestAttributeSource.java:926653,926948-928243,928255 +/lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/util/TestAttributeSource.java:926653,926791,926948-928243,928255 /lucene/java/branches/lucene_2_4/src/test/org/apache/lucene/util/TestAttributeSource.java:748824 /lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/util/TestAttributeSource.java:817269-818600,825998,829134,829881,831036 /lucene/java/branches/lucene_2_9_back_compat_tests/src/test/org/apache/lucene/util/TestAttributeSource.java:818601-821336 Propchange: lucene/java/branches/flex_1458/build.xml ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Sat Mar 27 21:21:26 2010 @@ -1,4 +1,4 @@ -/lucene/dev/trunk/lucene/build.xml:926653,926948-928243,928255 +/lucene/dev/trunk/lucene/build.xml:926653,926791,926948-928243,928255 /lucene/java/branches/lucene_2_9/build.xml:909334 /lucene/java/trunk/build.xml:916596,916685,916755,916769,917019,917148,917154,917161,917203-917204,919060,919119,919718,919731,919869,920237-920499,922013,922277,922525,922528,922583,922593,922602,922624,922797,922799,922806,922886-922887,923112,923238,924151,924207,924286,924321-924731,924781,925176-925462 /lucene/solr/branches/newtrunk/lucene/build.xml:926020-926289 Propchange: lucene/java/branches/flex_1458/contrib/ ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Sat Mar 27 21:21:26 2010 @@ -1,4 +1,4 @@ -/lucene/dev/trunk/lucene/contrib:926653,926948-928243,928255 +/lucene/dev/trunk/lucene/contrib:926653,926791,926948-928243,928255 /lucene/java/branches/lucene_2_4/contrib:748824 /lucene/java/branches/lucene_2_9/contrib:817269-818600,825998,829134,829816,829881,831036,896850,909334 /lucene/java/branches/lucene_2_9_back_compat_tests/contrib:818601-821336 Propchange: lucene/java/branches/flex_1458/contrib/CHANGES.txt ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Sat Mar 27 21:21:26 2010 @@ -1,4 +1,4 @@ -/lucene/dev/trunk/lucene/contrib/CHANGES.txt:926653,926948-928243,928255 +/lucene/dev/trunk/lucene/contrib/CHANGES.txt:926653,926791,926948-928243,928255 /lucene/java/branches/lucene_2_4/contrib/CHANGES.txt:748824 /lucene/java/branches/lucene_2_9/contrib/CHANGES.txt:817269-818600,825998,826775,829134,829816,829881,831036,896850 /lucene/java/branches/lucene_2_9_back_compat_tests/contrib/CHANGES.txt:818601-821336 Propchange: lucene/java/branches/flex_1458/contrib/highlighter/src/test/ ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Sat Mar 27 21:21:26 2010 @@ -1,4 +1,4 @@ -/lucene/dev/trunk/lucene/contrib/highlighter/src/test:926653,926948-928243,928255 +/lucene/dev/trunk/lucene/contrib/highlighter/src/test:926653,926791,926948-928243,928255 /lucene/java/branches/lucene_2_4/contrib/highlighter/src/test:748824 /lucene/java/branches/lucene_2_9/contrib/highlighter/src/test:817269-818600,825998,826775,829134,829816,829881,831036,896850,909334 /lucene/java/branches/lucene_2_9_back_compat_tests/contrib/highlighter/src/test:818601-821336 Propchange: lucene/java/branches/flex_1458/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Sat Mar 27 21:21:26 2010 @@ -1,4 +1,4 @@ -/lucene/dev/trunk/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java:926653,926948-928243,928255 +/lucene/dev/trunk/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java:926653,926791,926948-928243,928255 /lucene/java/branches/lucene_2_9/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java:909334 /lucene/java/trunk/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java:916596,916685,916755,916769,917019,917148,917154,917161,917203-917204,919060,919119,919718,919731,919869,920237-920499,922013,922277,922525,922528,922583,922593,922602,922624,922797,922799,922806,922886-922887,923112,923238,924151,924207,924286,924321-924731,924781,925176-925462 /lucene/solr/branches/newtrunk/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java:926020-926289 Propchange: lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/Tokenizer.java ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Sat Mar 27 21:21:26 2010 @@ -1,4 +1,4 @@ -/lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/Tokenizer.java:926653,926948-928243,928255 +/lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/Tokenizer.java:926653,926791,926948-928243,928255 /lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/analysis/Tokenizer.java:909334 /lucene/java/trunk/src/java/org/apache/lucene/analysis/Tokenizer.java:916596,916685,916755,916769,917019,917148,917154,917161,917203-917204,919060,919119,919718,919731,919869,920237-920499,922013,922277,922525,922528,922583,922593,922602,922624,922797,922799,922806,922886-922887,923112,923238,924151,924207,924286,924321-924731,924781,925176-925462 /lucene/solr/branches/newtrunk/lucene/src/java/org/apache/lucene/analysis/Tokenizer.java:926020-926289 Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/FreqProxFieldMergeState.java URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/FreqProxFieldMergeState.java?rev=928283&r1=928282&r2=928283&view=diff ============================================================================== --- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/FreqProxFieldMergeState.java (original) +++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/FreqProxFieldMergeState.java Sat Mar 27 21:21:26 2010 @@ -21,6 +21,8 @@ import java.io.IOException; import java.util.Comparator; import org.apache.lucene.util.BytesRef; +import org.apache.lucene.index.FreqProxTermsWriterPerField.FreqProxPostingsArray; + // TODO FI: some of this is "generic" to TermsHash* so we // should factor it out so other consumers don't have to // duplicate this code @@ -32,9 +34,10 @@ final class FreqProxFieldMergeState { final FreqProxTermsWriterPerField field; final int numPostings; private final ByteBlockPool bytePool; - final RawPostingList[] postings; - - private FreqProxTermsWriter.PostingList p; + final int[] termIDs; + final FreqProxPostingsArray postings; + int currentTermID; + final BytesRef text = new BytesRef(); private int postingUpto = -1; @@ -48,8 +51,9 @@ final class FreqProxFieldMergeState { public FreqProxFieldMergeState(FreqProxTermsWriterPerField field, Comparator termComp) { this.field = field; this.numPostings = field.termsHashPerField.numPostings; - this.postings = field.termsHashPerField.sortPostings(termComp); this.bytePool = field.perThread.termsHashPerThread.bytePool; + this.termIDs = field.termsHashPerField.sortPostings(termComp); + this.postings = (FreqProxPostingsArray) field.termsHashPerField.postingsArray; } boolean nextTerm() throws IOException { @@ -58,15 +62,16 @@ final class FreqProxFieldMergeState { return false; } - p = (FreqProxTermsWriter.PostingList) postings[postingUpto]; + currentTermID = termIDs[postingUpto]; docID = 0; // Get BytesRef - bytePool.setBytesRef(text, p.textStart); + final int textStart = postings.textStarts[currentTermID]; + bytePool.setBytesRef(text, textStart); - field.termsHashPerField.initReader(freq, p, 0); + field.termsHashPerField.initReader(freq, currentTermID, 0); if (!field.fieldInfo.omitTermFreqAndPositions) { - field.termsHashPerField.initReader(prox, p, 1); + field.termsHashPerField.initReader(prox, currentTermID, 1); } // Should always be true @@ -78,12 +83,12 @@ final class FreqProxFieldMergeState { public boolean nextDoc() throws IOException { if (freq.eof()) { - if (p.lastDocCode != -1) { + if (postings.lastDocCodes[currentTermID] != -1) { // Return last doc - docID = p.lastDocID; + docID = postings.lastDocIDs[currentTermID]; if (!field.omitTermFreqAndPositions) - termFreq = p.docFreq; - p.lastDocCode = -1; + termFreq = postings.docFreqs[currentTermID]; + postings.lastDocCodes[currentTermID] = -1; return true; } else // EOF @@ -101,7 +106,7 @@ final class FreqProxFieldMergeState { termFreq = freq.readVInt(); } - assert docID != p.lastDocID; + assert docID != postings.lastDocIDs[currentTermID]; return true; } Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/FreqProxTermsWriter.java URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/FreqProxTermsWriter.java?rev=928283&r1=928282&r2=928283&view=diff ============================================================================== --- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/FreqProxTermsWriter.java (original) +++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/FreqProxTermsWriter.java Sat Mar 27 21:21:26 2010 @@ -39,13 +39,6 @@ final class FreqProxTermsWriter extends } @Override - void createPostings(RawPostingList[] postings, int start, int count) { - final int end = start + count; - for(int i=start;i 0; + + FreqProxPostingsArray postings = (FreqProxPostingsArray) termsHashPerField.postingsArray; + + assert omitTermFreqAndPositions || postings.docFreqs[termID] > 0; if (omitTermFreqAndPositions) { - if (docState.docID != p.lastDocID) { - assert docState.docID > p.lastDocID; - termsHashPerField.writeVInt(0, p.lastDocCode); - p.lastDocCode = docState.docID - p.lastDocID; - p.lastDocID = docState.docID; + if (docState.docID != postings.lastDocIDs[termID]) { + assert docState.docID > postings.lastDocIDs[termID]; + termsHashPerField.writeVInt(0, postings.lastDocCodes[termID]); + postings.lastDocCodes[termID] = docState.docID - postings.lastDocIDs[termID]; + postings.lastDocIDs[termID] = docState.docID; } } else { - if (docState.docID != p.lastDocID) { - assert docState.docID > p.lastDocID; + if (docState.docID != postings.lastDocIDs[termID]) { + assert docState.docID > postings.lastDocIDs[termID]; // Term not yet seen in the current doc but previously // seen in other doc(s) since the last flush // Now that we know doc freq for previous doc, // write it & lastDocCode - if (1 == p.docFreq) - termsHashPerField.writeVInt(0, p.lastDocCode|1); + if (1 == postings.docFreqs[termID]) + termsHashPerField.writeVInt(0, postings.lastDocCodes[termID]|1); else { - termsHashPerField.writeVInt(0, p.lastDocCode); - termsHashPerField.writeVInt(0, p.docFreq); + termsHashPerField.writeVInt(0, postings.lastDocCodes[termID]); + termsHashPerField.writeVInt(0, postings.docFreqs[termID]); } - p.docFreq = 1; - p.lastDocCode = (docState.docID - p.lastDocID) << 1; - p.lastDocID = docState.docID; - writeProx(p, fieldState.position); + postings.docFreqs[termID] = 1; + postings.lastDocCodes[termID] = (docState.docID - postings.lastDocIDs[termID]) << 1; + postings.lastDocIDs[termID] = docState.docID; + writeProx(termID, fieldState.position); } else { - p.docFreq++; - writeProx(p, fieldState.position-p.lastPosition); + postings.docFreqs[termID]++; + writeProx(termID, fieldState.position-postings.lastPositions[termID]); } } } + + @Override + ParallelPostingsArray createPostingsArray(int size) { + return new FreqProxPostingsArray(size); + } + static final class FreqProxPostingsArray extends ParallelPostingsArray { + public FreqProxPostingsArray(int size) { + super(size); + docFreqs = new int[size]; + lastDocIDs = new int[size]; + lastDocCodes = new int[size]; + lastPositions = new int[size]; + } + + int docFreqs[]; // # times this term occurs in the current doc + int lastDocIDs[]; // Last docID where this term occurred + int lastDocCodes[]; // Code for prior doc + int lastPositions[]; // Last position where this term occurred + + @Override + ParallelPostingsArray resize(int newSize) { + FreqProxPostingsArray newArray = new FreqProxPostingsArray(newSize); + copy(this, newArray); + return newArray; + } + + void copy(FreqProxPostingsArray fromArray, FreqProxPostingsArray toArray) { + super.copy(fromArray, toArray); + System.arraycopy(fromArray.docFreqs, 0, toArray.docFreqs, 0, fromArray.docFreqs.length); + System.arraycopy(fromArray.lastDocIDs, 0, toArray.lastDocIDs, 0, fromArray.lastDocIDs.length); + System.arraycopy(fromArray.lastDocCodes, 0, toArray.lastDocCodes, 0, fromArray.lastDocCodes.length); + System.arraycopy(fromArray.lastPositions, 0, toArray.lastPositions, 0, fromArray.lastPositions.length); + } + + } + + @Override + int bytesPerPosting() { + return ParallelPostingsArray.BYTES_PER_POSTING + 4 * DocumentsWriter.INT_NUM_BYTE; + } + public void abort() {} } Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/TermVectorsTermsWriter.java URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/TermVectorsTermsWriter.java?rev=928283&r1=928282&r2=928283&view=diff ============================================================================== --- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/TermVectorsTermsWriter.java (original) +++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/TermVectorsTermsWriter.java Sat Mar 27 21:21:26 2010 @@ -48,13 +48,6 @@ final class TermVectorsTermsWriter exten } @Override - void createPostings(RawPostingList[] postings, int start, int count) { - final int end = start + count; - for(int i=start;i> threadsAndFields, final SegmentWriteState state) throws IOException { if (tvx != null) { @@ -290,15 +283,4 @@ final class TermVectorsTermsWriter exten finishDocument(this); } } - - static final class PostingList extends RawPostingList { - int freq; // How many times this term occurred in the current doc - int lastOffset; // Last offset we saw - int lastPosition; // Last position where this term occurred - } - - @Override - int bytesPerPosting() { - return RawPostingList.BYTES_SIZE + 3 * DocumentsWriter.INT_NUM_BYTE; - } } Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/TermVectorsTermsWriterPerField.java URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/TermVectorsTermsWriterPerField.java?rev=928283&r1=928282&r2=928283&view=diff ============================================================================== --- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/TermVectorsTermsWriterPerField.java (original) +++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/TermVectorsTermsWriterPerField.java Sat Mar 27 21:21:26 2010 @@ -126,10 +126,11 @@ final class TermVectorsTermsWriterPerFie assert perThread.vectorFieldsInOrder(fieldInfo); perThread.doc.addField(termsHashPerField.fieldInfo.number); + TermVectorsPostingsArray postings = (TermVectorsPostingsArray) termsHashPerField.postingsArray; // TODO: we may want to make this sort in same order // as Codec's terms dict? - final RawPostingList[] postings = termsHashPerField.sortPostings(BytesRef.getUTF8SortedAsUTF16Comparator()); + final int[] termIDs = termsHashPerField.sortPostings(BytesRef.getUTF8SortedAsUTF16Comparator()); tvf.writeVInt(numPostings); byte bits = 0x0; @@ -147,11 +148,11 @@ final class TermVectorsTermsWriterPerFie final ByteBlockPool termBytePool = perThread.termsHashPerThread.termBytePool; for(int j=0;j> threadsAndFields, SegmentWriteState state) { - - assert postingsFreeCount == postingsAllocCount: Thread.currentThread().getName() + ": postingsFreeCount=" + postingsFreeCount + " postingsAllocCount=" + postingsAllocCount + " consumer=" + consumer; - - final int newSize = ArrayUtil.getShrinkSize(postingsFreeList.length, postingsAllocCount, RamUsageEstimator.NUM_BYTES_OBJECT_REF); - if (newSize != postingsFreeList.length) { - RawPostingList[] newArray = new RawPostingList[newSize]; - System.arraycopy(postingsFreeList, 0, newArray, 0, postingsFreeCount); - postingsFreeList = newArray; - } - } - @Override synchronized void closeDocStore(SegmentWriteState state) throws IOException { consumer.closeDocStore(state); @@ -144,91 +115,12 @@ final class TermsHash extends InvertedDo consumer.flush(childThreadsAndFields, state); - shrinkFreePostings(threadsAndFields, state); - if (nextTermsHash != null) nextTermsHash.flush(nextThreadsAndFields, state); } @Override synchronized public boolean freeRAM() { - - if (!trackAllocations) - return false; - - boolean any; - final int numToFree; - if (postingsFreeCount >= postingsFreeChunk) - numToFree = postingsFreeChunk; - else - numToFree = postingsFreeCount; - any = numToFree > 0; - if (any) { - Arrays.fill(postingsFreeList, postingsFreeCount-numToFree, postingsFreeCount, null); - postingsFreeCount -= numToFree; - postingsAllocCount -= numToFree; - docWriter.bytesAllocated(-numToFree * bytesPerPosting); - any = true; - } - - if (nextTermsHash != null) - any |= nextTermsHash.freeRAM(); - - return any; - } - - synchronized public void recyclePostings(final RawPostingList[] postings, final int numPostings) { - - assert postings.length >= numPostings; - - // Move all Postings from this ThreadState back to our - // free list. We pre-allocated this array while we were - // creating Postings to make sure it's large enough - assert postingsFreeCount + numPostings <= postingsFreeList.length; - System.arraycopy(postings, 0, postingsFreeList, postingsFreeCount, numPostings); - postingsFreeCount += numPostings; - } - - synchronized public void getPostings(final RawPostingList[] postings) { - - assert docWriter.writer.testPoint("TermsHash.getPostings start"); - - assert postingsFreeCount <= postingsFreeList.length; - assert postingsFreeCount <= postingsAllocCount: "postingsFreeCount=" + postingsFreeCount + " postingsAllocCount=" + postingsAllocCount; - - final int numToCopy; - if (postingsFreeCount < postings.length) - numToCopy = postingsFreeCount; - else - numToCopy = postings.length; - final int start = postingsFreeCount-numToCopy; - assert start >= 0; - assert start + numToCopy <= postingsFreeList.length; - assert numToCopy <= postings.length; - System.arraycopy(postingsFreeList, start, - postings, 0, numToCopy); - - // Directly allocate the remainder if any - if (numToCopy != postings.length) { - final int extra = postings.length - numToCopy; - final int newPostingsAllocCount = postingsAllocCount + extra; - - consumer.createPostings(postings, numToCopy, extra); - assert docWriter.writer.testPoint("TermsHash.getPostings after create"); - postingsAllocCount += extra; - - if (trackAllocations) - docWriter.bytesAllocated(extra * bytesPerPosting); - - if (newPostingsAllocCount > postingsFreeList.length) - // Pre-allocate the postingsFreeList so it's large - // enough to hold all postings we've given out - postingsFreeList = new RawPostingList[ArrayUtil.oversize(newPostingsAllocCount, RamUsageEstimator.NUM_BYTES_OBJECT_REF)]; - } - - postingsFreeCount -= numToCopy; - - if (trackAllocations) - docWriter.bytesUsed(postings.length * bytesPerPosting); + return false; } } Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/TermsHashConsumer.java URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/TermsHashConsumer.java?rev=928283&r1=928282&r2=928283&view=diff ============================================================================== --- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/TermsHashConsumer.java (original) +++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/TermsHashConsumer.java Sat Mar 27 21:21:26 2010 @@ -22,8 +22,6 @@ import java.util.Collection; import java.util.Map; abstract class TermsHashConsumer { - abstract int bytesPerPosting(); - abstract void createPostings(RawPostingList[] postings, int start, int count); abstract TermsHashConsumerPerThread addThread(TermsHashPerThread perThread); abstract void flush(Map> threadsAndFields, final SegmentWriteState state) throws IOException; abstract void abort(); Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/TermsHashConsumerPerField.java URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/TermsHashConsumerPerField.java?rev=928283&r1=928282&r2=928283&view=diff ============================================================================== --- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/TermsHashConsumerPerField.java (original) +++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/TermsHashConsumerPerField.java Sat Mar 27 21:21:26 2010 @@ -31,7 +31,11 @@ abstract class TermsHashConsumerPerField abstract void finish() throws IOException; abstract void skippingLongTerm() throws IOException; abstract void start(Fieldable field); - abstract void newTerm(RawPostingList p) throws IOException; - abstract void addTerm(RawPostingList p) throws IOException; + abstract void newTerm(int termID) throws IOException; + abstract void addTerm(int termID) throws IOException; abstract int getStreamCount(); + + abstract ParallelPostingsArray createPostingsArray(int size); + abstract int bytesPerPosting(); + } Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/TermsHashPerField.java URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/TermsHashPerField.java?rev=928283&r1=928282&r2=928283&view=diff ============================================================================== --- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/TermsHashPerField.java (original) +++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/TermsHashPerField.java Sat Mar 27 21:21:26 2010 @@ -29,6 +29,7 @@ import org.apache.lucene.util.BytesRef; final class TermsHashPerField extends InvertedDocConsumerPerField { final TermsHashConsumerPerField consumer; + final TermsHashPerField nextPerField; final TermsHashPerThread perThread; final DocumentsWriter.DocState docState; @@ -50,17 +51,22 @@ final class TermsHashPerField extends In private int postingsHashSize = 4; private int postingsHashHalfSize = postingsHashSize/2; private int postingsHashMask = postingsHashSize-1; - private RawPostingList[] postingsHash = new RawPostingList[postingsHashSize]; - private RawPostingList p; + private int[] postingsHash; + + ParallelPostingsArray postingsArray; private final BytesRef utf8; private Comparator termComp; + private final int bytesPerPosting; + public TermsHashPerField(DocInverterPerField docInverterPerField, final TermsHashPerThread perThread, final TermsHashPerThread nextPerThread, final FieldInfo fieldInfo) { this.perThread = perThread; intPool = perThread.intPool; bytePool = perThread.bytePool; termBytePool = perThread.termBytePool; docState = perThread.docState; + postingsHash = new int[postingsHashSize]; + Arrays.fill(postingsHash, -1); fieldState = docInverterPerField.fieldState; this.consumer = perThread.consumer.addField(this, fieldInfo); @@ -72,6 +78,21 @@ final class TermsHashPerField extends In nextPerField = (TermsHashPerField) nextPerThread.addField(docInverterPerField, fieldInfo); else nextPerField = null; + + // +3: Posting is referenced by hash, which + // targets 25-50% fill factor; approximate this + // as 3X # pointers + bytesPerPosting = consumer.bytesPerPosting() + 3*DocumentsWriter.INT_NUM_BYTE; + } + + void initPostingsArray() { + assert postingsArray == null; + + postingsArray = consumer.createPostingsArray(postingsHashSize); + + if (perThread.termsHash.trackAllocations) { + perThread.termsHash.docWriter.bytesAllocated(bytesPerPosting * postingsHashSize); + } } void shrinkHash(int targetSize) { @@ -85,7 +106,9 @@ final class TermsHashPerField extends In } if (newSize != postingsHash.length) { - postingsHash = new RawPostingList[newSize]; + postingsHash = new int[newSize]; + Arrays.fill(postingsHash, -1); + postingsArray = null; postingsHashSize = newSize; postingsHashHalfSize = newSize/2; postingsHashMask = newSize-1; @@ -97,8 +120,7 @@ final class TermsHashPerField extends In compactPostings(); assert numPostings <= postingsHash.length; if (numPostings > 0) { - perThread.termsHash.recyclePostings(postingsHash, numPostings); - Arrays.fill(postingsHash, 0, numPostings, null); + Arrays.fill(postingsHash, 0, numPostings, -1); numPostings = 0; } postingsCompacted = false; @@ -112,23 +134,34 @@ final class TermsHashPerField extends In if (nextPerField != null) nextPerField.abort(); } + + private void growParallelPostingsArray() { + int oldSize = postingsArray.byteStarts.length; + int newSize = (int) (oldSize * 1.5); + this.postingsArray = this.postingsArray.resize(newSize); + + if (perThread.termsHash.trackAllocations) { + perThread.termsHash.docWriter.bytesAllocated(bytesPerPosting * (newSize - oldSize)); + } + } - public void initReader(ByteSliceReader reader, RawPostingList p, int stream) { + public void initReader(ByteSliceReader reader, int termID, int stream) { assert stream < streamCount; - final int[] ints = intPool.buffers[p.intStart >> DocumentsWriter.INT_BLOCK_SHIFT]; - final int upto = p.intStart & DocumentsWriter.INT_BLOCK_MASK; + int intStart = postingsArray.intStarts[termID]; + final int[] ints = intPool.buffers[intStart >> DocumentsWriter.INT_BLOCK_SHIFT]; + final int upto = intStart & DocumentsWriter.INT_BLOCK_MASK; reader.init(bytePool, - p.byteStart+stream*ByteBlockPool.FIRST_LEVEL_SIZE, + postingsArray.byteStarts[termID]+stream*ByteBlockPool.FIRST_LEVEL_SIZE, ints[upto+stream]); } private synchronized void compactPostings() { int upto = 0; for(int i=0;i termComp) { + public int[] sortPostings(Comparator termComp) { this.termComp = termComp; compactPostings(); quickSort(postingsHash, 0, numPostings-1); return postingsHash; } - void quickSort(RawPostingList[] postings, int lo, int hi) { + void quickSort(int[] termIDs, int lo, int hi) { if (lo >= hi) return; else if (hi == 1+lo) { - if (comparePostings(postings[lo], postings[hi]) > 0) { - final RawPostingList tmp = postings[lo]; - postings[lo] = postings[hi]; - postings[hi] = tmp; + if (comparePostings(termIDs[lo], termIDs[hi]) > 0) { + final int tmp = termIDs[lo]; + termIDs[lo] = termIDs[hi]; + termIDs[hi] = tmp; } return; } int mid = (lo + hi) >>> 1; - if (comparePostings(postings[lo], postings[mid]) > 0) { - RawPostingList tmp = postings[lo]; - postings[lo] = postings[mid]; - postings[mid] = tmp; + if (comparePostings(termIDs[lo], termIDs[mid]) > 0) { + int tmp = termIDs[lo]; + termIDs[lo] = termIDs[mid]; + termIDs[mid] = tmp; } - if (comparePostings(postings[mid], postings[hi]) > 0) { - RawPostingList tmp = postings[mid]; - postings[mid] = postings[hi]; - postings[hi] = tmp; - - if (comparePostings(postings[lo], postings[mid]) > 0) { - RawPostingList tmp2 = postings[lo]; - postings[lo] = postings[mid]; - postings[mid] = tmp2; + if (comparePostings(termIDs[mid], termIDs[hi]) > 0) { + int tmp = termIDs[mid]; + termIDs[mid] = termIDs[hi]; + termIDs[hi] = tmp; + + if (comparePostings(termIDs[lo], termIDs[mid]) > 0) { + int tmp2 = termIDs[lo]; + termIDs[lo] = termIDs[mid]; + termIDs[mid] = tmp2; } } @@ -184,51 +217,52 @@ final class TermsHashPerField extends In if (left >= right) return; - RawPostingList partition = postings[mid]; + int partition = termIDs[mid]; for (; ;) { - while (comparePostings(postings[right], partition) > 0) + while (comparePostings(termIDs[right], partition) > 0) --right; - while (left < right && comparePostings(postings[left], partition) <= 0) + while (left < right && comparePostings(termIDs[left], partition) <= 0) ++left; if (left < right) { - RawPostingList tmp = postings[left]; - postings[left] = postings[right]; - postings[right] = tmp; + int tmp = termIDs[left]; + termIDs[left] = termIDs[right]; + termIDs[right] = tmp; --right; } else { break; } } - quickSort(postings, lo, left); - quickSort(postings, left + 1, hi); + quickSort(termIDs, lo, left); + quickSort(termIDs, left + 1, hi); } /** Compares term text for two Posting instance and * returns -1 if p1 < p2; 1 if p1 > p2; else 0. */ - int comparePostings(RawPostingList p1, RawPostingList p2) { + int comparePostings(int term1, int term2) { - if (p1 == p2) { + if (term1 == term2) { // Our quicksort does this, eg during partition return 0; } - termBytePool.setBytesRef(perThread.tr1, p1.textStart); - termBytePool.setBytesRef(perThread.tr2, p2.textStart); + termBytePool.setBytesRef(perThread.tr1, postingsArray.textStarts[term1]); + termBytePool.setBytesRef(perThread.tr2, postingsArray.textStarts[term2]); return termComp.compare(perThread.tr1, perThread.tr2); } /** Test whether the text for current RawPostingList p equals * current tokenText in utf8. */ - private boolean postingEquals() { - - final byte[] text = termBytePool.buffers[p.textStart >> DocumentsWriter.BYTE_BLOCK_SHIFT]; + private boolean postingEquals(final int termID) { + final int textStart = postingsArray.textStarts[termID]; + final byte[] text = termBytePool.buffers[textStart >> DocumentsWriter.BYTE_BLOCK_SHIFT]; assert text != null; - int pos = p.textStart & DocumentsWriter.BYTE_BLOCK_MASK; + + int pos = textStart & DocumentsWriter.BYTE_BLOCK_MASK; final int len; if ((text[pos] & 0x80) == 0) { @@ -267,6 +301,9 @@ final class TermsHashPerField extends In } else { throw new IllegalArgumentException("Could not find a term attribute (that implements TermToBytesRefAttribute) in the TokenStream"); } + if (postingsArray == null) { + initPostingsArray(); + } consumer.start(f); if (nextPerField != null) { nextPerField.start(f); @@ -285,7 +322,6 @@ final class TermsHashPerField extends In // because token text has already been "interned" into // textStart, so we hash by textStart public void add(int textStart) throws IOException { - int code = textStart; int hashPos = code & postingsHashMask; @@ -293,37 +329,39 @@ final class TermsHashPerField extends In assert !postingsCompacted; // Locate RawPostingList in hash - p = postingsHash[hashPos]; + int termID = postingsHash[hashPos]; - if (p != null && p.textStart != textStart) { + if (termID != -1 && postingsArray.textStarts[termID] != textStart) { // Conflict: keep searching different locations in // the hash table. final int inc = ((code>>8)+code)|1; do { code += inc; hashPos = code & postingsHashMask; - p = postingsHash[hashPos]; - } while (p != null && p.textStart != textStart); + termID = postingsHash[hashPos]; + } while (termID != -1 && postingsArray.textStarts[termID] != textStart); } - if (p == null) { + if (termID == -1) { // First time we are seeing this token since we last // flushed the hash. - // Refill? - if (0 == perThread.freePostingsCount) - perThread.morePostings(); - - // Pull next free RawPostingList from free list - p = perThread.freePostings[--perThread.freePostingsCount]; - assert p != null; + // New posting + termID = numPostings++; + if (termID >= postingsArray.textStarts.length) { + growParallelPostingsArray(); + } + if (perThread.termsHash.trackAllocations) { + perThread.termsHash.docWriter.bytesUsed(bytesPerPosting); + } + + assert termID >= 0; - p.textStart = textStart; + postingsArray.textStarts[termID] = textStart; - assert postingsHash[hashPos] == null; - postingsHash[hashPos] = p; - numPostings++; + assert postingsHash[hashPos] == -1; + postingsHash[hashPos] = termID; if (numPostings == postingsHashHalfSize) rehashPostings(2*postingsHashSize); @@ -339,20 +377,21 @@ final class TermsHashPerField extends In intUptoStart = intPool.intUpto; intPool.intUpto += streamCount; - p.intStart = intUptoStart + intPool.intOffset; + postingsArray.intStarts[termID] = intUptoStart + intPool.intOffset; for(int i=0;i> DocumentsWriter.INT_BLOCK_SHIFT]; - intUptoStart = p.intStart & DocumentsWriter.INT_BLOCK_MASK; - consumer.addTerm(p); + int intStart = postingsArray.intStarts[termID]; + intUptos = intPool.buffers[intStart >> DocumentsWriter.INT_BLOCK_SHIFT]; + intUptoStart = intStart & DocumentsWriter.INT_BLOCK_MASK; + consumer.addTerm(termID); } } @@ -371,20 +410,20 @@ final class TermsHashPerField extends In int hashPos = code & postingsHashMask; // Locate RawPostingList in hash - p = postingsHash[hashPos]; + int termID = postingsHash[hashPos]; - if (p != null && !postingEquals()) { + if (termID != -1 && !postingEquals(termID)) { // Conflict: keep searching different locations in // the hash table. final int inc = ((code>>8)+code)|1; do { code += inc; hashPos = code & postingsHashMask; - p = postingsHash[hashPos]; - } while (p != null && !postingEquals()); + termID = postingsHash[hashPos]; + } while (termID != -1 && !postingEquals(termID)); } - if (p == null) { + if (termID == -1) { // First time we are seeing this token since we last // flushed the hash. @@ -414,18 +453,23 @@ final class TermsHashPerField extends In bytePool.nextBuffer(); } - // Refill? - if (0 == perThread.freePostingsCount) { - perThread.morePostings(); + // New posting + termID = numPostings++; + if (termID >= postingsArray.textStarts.length) { + growParallelPostingsArray(); + } + if (perThread.termsHash.trackAllocations) { + perThread.termsHash.docWriter.bytesUsed(bytesPerPosting); } - // Pull next free RawPostingList from free list - p = perThread.freePostings[--perThread.freePostingsCount]; - assert p != null; + assert termID != -1; + assert postingsHash[hashPos] == -1; + + postingsHash[hashPos] = termID; final byte[] text = bytePool.buffer; final int textUpto = bytePool.byteUpto; - p.textStart = textUpto + bytePool.byteOffset; + postingsArray.textStarts[termID] = textUpto + bytePool.byteOffset; // We first encode the length, followed by the UTF8 // bytes. Length is encoded as vInt, but will consume @@ -446,10 +490,6 @@ final class TermsHashPerField extends In System.arraycopy(utf8.bytes, 0, text, textUpto+2, utf8.length); } - assert postingsHash[hashPos] == null; - postingsHash[hashPos] = p; - numPostings++; - if (numPostings == postingsHashHalfSize) { rehashPostings(2*postingsHashSize); } @@ -467,25 +507,25 @@ final class TermsHashPerField extends In intUptoStart = intPool.intUpto; intPool.intUpto += streamCount; - p.intStart = intUptoStart + intPool.intOffset; + postingsArray.intStarts[termID] = intUptoStart + intPool.intOffset; for(int i=0;i> DocumentsWriter.INT_BLOCK_SHIFT]; - intUptoStart = p.intStart & DocumentsWriter.INT_BLOCK_MASK; - consumer.addTerm(p); + final int intStart = postingsArray.intStarts[termID]; + intUptos = intPool.buffers[intStart >> DocumentsWriter.INT_BLOCK_SHIFT]; + intUptoStart = intStart & DocumentsWriter.INT_BLOCK_MASK; + consumer.addTerm(termID); } if (doNextCall) - nextPerField.add(p.textStart); + nextPerField.add(postingsArray.textStarts[termID]); } int[] intUptos; @@ -535,14 +575,16 @@ final class TermsHashPerField extends In final int newMask = newSize-1; - RawPostingList[] newHash = new RawPostingList[newSize]; + int[] newHash = new int[newSize]; + Arrays.fill(newHash, -1); for(int i=0;i> DocumentsWriter.BYTE_BLOCK_SHIFT]; + final int textStart = postingsArray.textStarts[termID]; + final int start = textStart & DocumentsWriter.BYTE_BLOCK_MASK; + final byte[] text = bytePool.buffers[textStart >> DocumentsWriter.BYTE_BLOCK_SHIFT]; code = 0; final int len; @@ -561,19 +603,19 @@ final class TermsHashPerField extends In code = (code*31) + text[pos++]; } } else { - code = p0.textStart; + code = postingsArray.textStarts[termID]; } int hashPos = code & newMask; assert hashPos >= 0; - if (newHash[hashPos] != null) { + if (newHash[hashPos] != -1) { final int inc = ((code>>8)+code)|1; do { code += inc; hashPos = code & newMask; - } while (newHash[hashPos] != null); + } while (newHash[hashPos] != -1); } - newHash[hashPos] = p0; + newHash[hashPos] = termID; } } Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/TermsHashPerThread.java URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/TermsHashPerThread.java?rev=928283&r1=928282&r2=928283&view=diff ============================================================================== --- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/TermsHashPerThread.java (original) +++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/TermsHashPerThread.java Sat Mar 27 21:21:26 2010 @@ -36,9 +36,6 @@ final class TermsHashPerThread extends I final boolean primary; final DocumentsWriter.DocState docState; - final RawPostingList freePostings[] = new RawPostingList[256]; - int freePostingsCount; - // Used when comparing postings via termRefComp, in TermsHashPerField final BytesRef tr1 = new BytesRef(); final BytesRef tr2 = new BytesRef(); @@ -100,20 +97,6 @@ final class TermsHashPerThread extends I nextPerThread.abort(); } - // perField calls this when it needs more postings: - void morePostings() throws IOException { - assert freePostingsCount == 0; - termsHash.getPostings(freePostings); - freePostingsCount = freePostings.length; - assert noNullPostings(freePostings, freePostingsCount, "consumer=" + consumer); - } - - private static boolean noNullPostings(RawPostingList[] postings, int count, String details) { - for(int i=0;i filenames = new ArrayList(); - IndexFileNameFilter filter = IndexFileNameFilter.getFilter(); - - for (String name : listAll()) - if (filter.accept(null, name)) - filenames.add(name); - - copyTo(to, filenames); - */ - copyTo(to, asList(listAll())); + copyTo(to, Arrays.asList(listAll())); } /** Propchange: lucene/java/branches/flex_1458/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Sat Mar 27 21:21:26 2010 @@ -1,4 +1,4 @@ -/lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java:926653,926948-928243,928255 +/lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java:926653,926791,926948-928243,928255 /lucene/java/branches/lucene_2_4/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java:748824 /lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java:829134,829881,831036,896850,909334 /lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java:880793,896906 Propchange: lucene/java/branches/flex_1458/src/test/org/apache/lucene/document/TestDateTools.java ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Sat Mar 27 21:21:26 2010 @@ -1,4 +1,4 @@ -/lucene/dev/trunk/lucene/src/test/org/apache/lucene/document/TestDateTools.java:926653,926948-928243,928255 +/lucene/dev/trunk/lucene/src/test/org/apache/lucene/document/TestDateTools.java:926653,926791,926948-928243,928255 /lucene/java/branches/lucene_2_4/src/test/org/apache/lucene/document/TestDateTools.java:748824 /lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/document/TestDateTools.java:829134,829881,831036,896850,909334 /lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/document/TestDateTools.java:880793,896906 Propchange: lucene/java/branches/flex_1458/src/test/org/apache/lucene/document/TestNumberTools.java ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Sat Mar 27 21:21:26 2010 @@ -1,4 +1,4 @@ -/lucene/dev/trunk/lucene/src/test/org/apache/lucene/document/TestNumberTools.java:926653,926948-928243,928255 +/lucene/dev/trunk/lucene/src/test/org/apache/lucene/document/TestNumberTools.java:926653,926791,926948-928243,928255 /lucene/java/branches/lucene_2_4/src/test/org/apache/lucene/document/TestNumberTools.java:748824 /lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/document/TestNumberTools.java:829134,829881,831036,896850,909334 /lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/document/TestNumberTools.java:880793,896906 Propchange: lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Sat Mar 27 21:21:26 2010 @@ -1,4 +1,4 @@ -/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:926653,926948-928243,928255 +/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:926653,926791,926948-928243,928255 /lucene/java/branches/lucene_2_4/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:748824 /lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:829134,829881,831036,896850,909334 /lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:880793,896906 Propchange: lucene/java/branches/flex_1458/src/test/org/apache/lucene/util/TestAttributeSource.java ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Sat Mar 27 21:21:26 2010 @@ -1,4 +1,4 @@ -/lucene/dev/trunk/lucene/src/test/org/apache/lucene/util/TestAttributeSource.java:926653,926948-928243,928255 +/lucene/dev/trunk/lucene/src/test/org/apache/lucene/util/TestAttributeSource.java:926653,926791,926948-928243,928255 /lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/util/TestAttributeSource.java:909334 /lucene/java/trunk/src/test/org/apache/lucene/util/TestAttributeSource.java:916596,916685,916755,916769,917019,917148,917154,917161,917203-917204,919060,919119,919718,919731,919869,920237-920499,922013,922277,922525,922528,922583,922593,922602,922624,922797,922799,922806,922886-922887,923112,923238,924151,924207,924286,924321-924731,924781,925176-925462 /lucene/solr/branches/newtrunk/lucene/src/test/org/apache/lucene/util/TestAttributeSource.java:926020-926289