lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sim...@apache.org
Subject svn commit: r1147873 [8/13] - in /lucene/dev/branches/positions: ./ dev-tools/eclipse/ dev-tools/idea/.idea/ dev-tools/idea/lucene/contrib/ dev-tools/idea/lucene/contrib/demo/ dev-tools/idea/lucene/contrib/memory/ dev-tools/idea/lucene/contrib/querypar...
Date Mon, 18 Jul 2011 13:35:33 GMT
Modified: lucene/dev/branches/positions/lucene/src/site/build/site/fileformats.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/lucene/src/site/build/site/fileformats.html?rev=1147873&r1=1147872&r2=1147873&view=diff
==============================================================================
--- lucene/dev/branches/positions/lucene/src/site/build/site/fileformats.html (original)
+++ lucene/dev/branches/positions/lucene/src/site/build/site/fileformats.html Mon Jul 18 13:34:29 2011
@@ -168,9 +168,6 @@ document.write("Last Published: " + docu
 <a href="api/contrib-spellchecker/index.html">Spellchecker</a>
 </div>
 <div class="menuitem">
-<a href="api/contrib-wordnet/index.html">Wordnet</a>
-</div>
-<div class="menuitem">
 <a href="api/contrib-xml-query-parser/index.html">XML Query Parser</a>
 </div>
 </div>
@@ -415,10 +412,14 @@ document.write("Last Published: " + docu
             to stored fields file, previously they were stored in
             text format only.
            </p>
+<p>
+            In version 3.4, fields can omit position data while
+            still indexing term frequencies.
+        </p>
 </div>
 
         
-<a name="N1003A"></a><a name="Definitions"></a>
+<a name="N1003D"></a><a name="Definitions"></a>
 <h2 class="boxed">Definitions</h2>
 <div class="section">
 <p>
@@ -459,7 +460,7 @@ document.write("Last Published: " + docu
                 strings, the first naming the field, and the second naming text
                 within the field.
             </p>
-<a name="N1005A"></a><a name="Inverted Indexing"></a>
+<a name="N1005D"></a><a name="Inverted Indexing"></a>
 <h3 class="boxed">Inverted Indexing</h3>
 <p>
                     The index stores statistics about terms in order
@@ -469,7 +470,7 @@ document.write("Last Published: " + docu
                     it.  This is the inverse of the natural relationship, in which
                     documents list terms.
                 </p>
-<a name="N10066"></a><a name="Types of Fields"></a>
+<a name="N10069"></a><a name="Types of Fields"></a>
 <h3 class="boxed">Types of Fields</h3>
 <p>
                     In Lucene, fields may be <i>stored</i>, in which
@@ -483,7 +484,7 @@ document.write("Last Published: " + docu
                     to be indexed literally.
                 </p>
 <p>See the <a href="api/core/org/apache/lucene/document/Field.html">Field</a> java docs for more information on Fields.</p>
-<a name="N10083"></a><a name="Segments"></a>
+<a name="N10086"></a><a name="Segments"></a>
 <h3 class="boxed">Segments</h3>
 <p>
                     Lucene indexes may be composed of multiple sub-indexes, or
@@ -509,7 +510,7 @@ document.write("Last Published: " + docu
                     Searches may involve multiple segments and/or multiple indexes, each
                     index potentially composed of a set of segments.
                 </p>
-<a name="N100A1"></a><a name="Document Numbers"></a>
+<a name="N100A4"></a><a name="Document Numbers"></a>
 <h3 class="boxed">Document Numbers</h3>
 <p>
                     Internally, Lucene refers to documents by an integer <i>document
@@ -564,7 +565,7 @@ document.write("Last Published: " + docu
 </div>
 
         
-<a name="N100C8"></a><a name="Overview"></a>
+<a name="N100CB"></a><a name="Overview"></a>
 <h2 class="boxed">Overview</h2>
 <div class="section">
 <p>
@@ -611,7 +612,7 @@ document.write("Last Published: " + docu
 <p>Term Frequency
                         data. For each term in the dictionary, the numbers of all the
                         documents that contain that term, and the frequency of the term in
-                        that document if omitTf is false.
+                        that document, unless frequencies are omitted (IndexOptions.DOCS_ONLY)
                     </p>
                 
 </li>
@@ -622,8 +623,7 @@ document.write("Last Published: " + docu
 <p>Term Proximity
                         data. For each term in the dictionary, the positions that the term
                         occurs in each document.  Note that this will
-                        not exist if all fields in all documents set
-                        omitTf to true.
+                        not exist if all fields in all documents omit position data.
                     </p>
                 
 </li>
@@ -663,7 +663,7 @@ document.write("Last Published: " + docu
 </div>
 
         
-<a name="N1010B"></a><a name="File Naming"></a>
+<a name="N1010E"></a><a name="File Naming"></a>
 <h2 class="boxed">File Naming</h2>
 <div class="section">
 <p>
@@ -690,7 +690,7 @@ document.write("Last Published: " + docu
             </p>
 </div>
       
-<a name="N1011A"></a><a name="file-names"></a>
+<a name="N1011D"></a><a name="file-names"></a>
 <h2 class="boxed">Summary of File Extensions</h2>
 <div class="section">
 <p>The following table summarizes the names and extensions of the files in Lucene:
@@ -840,10 +840,10 @@ document.write("Last Published: " + docu
 </div>
 
         
-<a name="N10212"></a><a name="Primitive Types"></a>
+<a name="N10215"></a><a name="Primitive Types"></a>
 <h2 class="boxed">Primitive Types</h2>
 <div class="section">
-<a name="N10217"></a><a name="Byte"></a>
+<a name="N1021A"></a><a name="Byte"></a>
 <h3 class="boxed">Byte</h3>
 <p>
                     The most primitive type
@@ -851,7 +851,7 @@ document.write("Last Published: " + docu
                     other data types are defined as sequences
                     of bytes, so file formats are byte-order independent.
                 </p>
-<a name="N10220"></a><a name="UInt32"></a>
+<a name="N10223"></a><a name="UInt32"></a>
 <h3 class="boxed">UInt32</h3>
 <p>
                     32-bit unsigned integers are written as four
@@ -861,7 +861,7 @@ document.write("Last Published: " + docu
                     UInt32    --&gt; &lt;Byte&gt;<sup>4</sup>
                 
 </p>
-<a name="N1022F"></a><a name="Uint64"></a>
+<a name="N10232"></a><a name="Uint64"></a>
 <h3 class="boxed">Uint64</h3>
 <p>
                     64-bit unsigned integers are written as eight
@@ -870,7 +870,7 @@ document.write("Last Published: " + docu
 <p>UInt64    --&gt; &lt;Byte&gt;<sup>8</sup>
                 
 </p>
-<a name="N1023E"></a><a name="VInt"></a>
+<a name="N10241"></a><a name="VInt"></a>
 <h3 class="boxed">VInt</h3>
 <p>
                     A variable-length format for positive integers is
@@ -1420,13 +1420,13 @@ document.write("Last Published: " + docu
                     This provides compression while still being
                     efficient to decode.
                 </p>
-<a name="N10523"></a><a name="Chars"></a>
+<a name="N10526"></a><a name="Chars"></a>
 <h3 class="boxed">Chars</h3>
 <p>
                     Lucene writes unicode
                     character sequences as UTF-8 encoded bytes.
                 </p>
-<a name="N1052C"></a><a name="String"></a>
+<a name="N1052F"></a><a name="String"></a>
 <h3 class="boxed">String</h3>
 <p>
 		    Lucene writes strings as UTF-8 encoded bytes.
@@ -1439,10 +1439,10 @@ document.write("Last Published: " + docu
 </div>
 
         
-<a name="N10539"></a><a name="Compound Types"></a>
+<a name="N1053C"></a><a name="Compound Types"></a>
 <h2 class="boxed">Compound Types</h2>
 <div class="section">
-<a name="N1053E"></a><a name="MapStringString"></a>
+<a name="N10541"></a><a name="MapStringString"></a>
 <h3 class="boxed">Map&lt;String,String&gt;</h3>
 <p>
 		    In a couple places Lucene stores a Map
@@ -1455,13 +1455,13 @@ document.write("Last Published: " + docu
 </div>
 
         
-<a name="N1054E"></a><a name="Per-Index Files"></a>
+<a name="N10551"></a><a name="Per-Index Files"></a>
 <h2 class="boxed">Per-Index Files</h2>
 <div class="section">
 <p>
                 The files in this section exist one-per-index.
             </p>
-<a name="N10556"></a><a name="Segments File"></a>
+<a name="N10559"></a><a name="Segments File"></a>
 <h3 class="boxed">Segments File</h3>
 <p>
                     The active segments in the index are stored in the
@@ -1616,7 +1616,7 @@ document.write("Last Published: " + docu
 		</p>
 <p>
 		    HasProx is 1 if any fields in this segment have
-		    omitTf set to false; else, it's 0.
+		    position data (IndexOptions.DOCS_AND_FREQS_AND_POSITIONS); else, it's 0.
 		</p>
 <p>
 		    CommitUserData stores an optional user-supplied
@@ -1634,7 +1634,7 @@ document.write("Last Published: " + docu
 <p> HasVectors is 1 if this segment stores term vectors,
             else it's 0.
                 </p>
-<a name="N105E1"></a><a name="Lock File"></a>
+<a name="N105E4"></a><a name="Lock File"></a>
 <h3 class="boxed">Lock File</h3>
 <p>
                     The write lock, which is stored in the index
@@ -1648,14 +1648,14 @@ document.write("Last Published: " + docu
                     documents).  This lock file ensures that only one
                     writer is modifying the index at a time.
                 </p>
-<a name="N105EA"></a><a name="Deletable File"></a>
+<a name="N105ED"></a><a name="Deletable File"></a>
 <h3 class="boxed">Deletable File</h3>
 <p>
                     A writer dynamically computes
                     the files that are deletable, instead, so no file
                     is written.
                 </p>
-<a name="N105F3"></a><a name="Compound Files"></a>
+<a name="N105F6"></a><a name="Compound Files"></a>
 <h3 class="boxed">Compound Files</h3>
 <p>Starting with Lucene 1.4 the compound file format became default. This
                     is simply a container for all files described in the next section
@@ -1684,14 +1684,14 @@ document.write("Last Published: " + docu
 </div>
 
         
-<a name="N10624"></a><a name="Per-Segment Files"></a>
+<a name="N10627"></a><a name="Per-Segment Files"></a>
 <h2 class="boxed">Per-Segment Files</h2>
 <div class="section">
 <p>
                 The remaining files are all per-segment, and are
                 thus defined by suffix.
             </p>
-<a name="N1062C"></a><a name="Fields"></a>
+<a name="N1062F"></a><a name="Fields"></a>
 <h3 class="boxed">Fields</h3>
 <p>
                     
@@ -1744,12 +1744,16 @@ document.write("Last Published: " + docu
 <li>If the fifth lowest-order bit is set (0x10), norms are omitted for the indexed field.</li>
                         
 <li>If the sixth lowest-order bit is set (0x20), payloads are stored for the indexed field.</li>
+                        
+<li>If the seventh lowest-order bit is set (0x40), term frequencies and positions omitted for the indexed field.</li>
+                        
+<li>If the eighth lowest-order bit is set (0x80), positions are omitted for the indexed field.</li>
                     
 </ul>
                 
 </p>
 <p>
-		   FNMVersion (added in 2.9) is always -2.
+		   FNMVersion (added in 2.9) is -2 for indexes from 2.9 - 3.3. It is -3 for indexes in Lucene 3.4+
 		</p>
 <p>
                     Fields are numbered by their order in this file. Thus field zero is
@@ -1901,7 +1905,7 @@ document.write("Last Published: " + docu
 </li>
                 
 </ol>
-<a name="N106E7"></a><a name="Term Dictionary"></a>
+<a name="N106F0"></a><a name="Term Dictionary"></a>
 <h3 class="boxed">Term Dictionary</h3>
 <p>
                     The term dictionary is represented as two files:
@@ -2005,7 +2009,7 @@ document.write("Last Published: " + docu
                             file. In particular, it is the difference between the position of
                             this term's data in that file and the position of the previous
                             term's data (or zero, for the first term in the file.  For fields
-			    with omitTf true, this will be 0 since
+			                that omit position data, this will be 0 since
                             prox information is not stored.
                         </p>
                         
@@ -2093,12 +2097,12 @@ document.write("Last Published: " + docu
 </li>
                 
 </ol>
-<a name="N1076B"></a><a name="Frequencies"></a>
+<a name="N10774"></a><a name="Frequencies"></a>
 <h3 class="boxed">Frequencies</h3>
 <p>
                     The .frq file contains the lists of documents
                     which contain each term, along with the frequency of the term in that
-                    document (if omitTf is false).
+                    document (except when frequencies are omitted: IndexOptions.DOCS_ONLY).
                 </p>
 <p>FreqFile (.frq) --&gt;
                     &lt;TermFreqs, SkipData&gt;
@@ -2138,26 +2142,26 @@ document.write("Last Published: " + docu
 <p>TermFreq
                     entries are ordered by increasing document number.
                 </p>
-<p>DocDelta: if omitTf is false, this determines both
+<p>DocDelta: if frequencies are indexed, this determines both
                     the document number and the frequency. In
                     particular, DocDelta/2 is the difference between
                     this document number and the previous document
                     number (or zero when this is the first document in
                     a TermFreqs). When DocDelta is odd, the frequency
                     is one. When DocDelta is even, the frequency is
-                    read as another VInt.  If omitTf is true, DocDelta
+                    read as another VInt.  If frequencies are omitted, DocDelta
                     contains the gap (not multiplied by 2) between
                     document numbers and no frequency information is
                     stored.
                 </p>
 <p>For example, the TermFreqs for a term which occurs
                     once in document seven and three times in document
-                    eleven, with omitTf false, would be the following
+                    eleven, with frequencies indexed, would be the following
                     sequence of VInts:
                 </p>
 <p>15, 8, 3
                 </p>
-<p> If omitTf were true it would be this sequence
+<p> If frequencies were omitted (IndexOptions.DOCS_ONLY) it would be this sequence
 		of VInts instead:
 		  </p>
 <p>
@@ -2221,14 +2225,14 @@ document.write("Last Published: " + docu
                    entry in level-1. In the example has entry 15 on level 1 a pointer to entry 15 on level 0 and entry 31 on level 1 a pointer
                    to entry 31 on level 0.                   
                 </p>
-<a name="N107F3"></a><a name="Positions"></a>
+<a name="N107FC"></a><a name="Positions"></a>
 <h3 class="boxed">Positions</h3>
 <p>
                     The .prx file contains the lists of positions that
                     each term occurs at within documents.  Note that
-                    fields with omitTf true do not store
+                    fields omitting positional data do not store
                     anything into this file, and if all fields in the
-                    index have omitTf true then the .prx file will not
+                    index omit positional data then the .prx file will not
                     exist.
                 </p>
 <p>ProxFile (.prx) --&gt;
@@ -2291,7 +2295,7 @@ document.write("Last Published: " + docu
                     Payload. If PayloadLength is not stored, then this Payload has the same
                     length as the Payload at the previous position.
                 </p>
-<a name="N1082F"></a><a name="Normalization Factors"></a>
+<a name="N10838"></a><a name="Normalization Factors"></a>
 <h3 class="boxed">Normalization Factors</h3>
 <p>There's a single .nrm file containing all norms:
                 </p>
@@ -2371,7 +2375,7 @@ document.write("Last Published: " + docu
                 </p>
 <p>Separate norm files are created (when adequate) for both compound and non compound segments.
                 </p>
-<a name="N10880"></a><a name="Term Vectors"></a>
+<a name="N10889"></a><a name="Term Vectors"></a>
 <h3 class="boxed">Term Vectors</h3>
 <p>
 		  Term Vector support is an optional on a field by
@@ -2507,7 +2511,7 @@ document.write("Last Published: " + docu
 </li>
                 
 </ol>
-<a name="N1091C"></a><a name="Deleted Documents"></a>
+<a name="N10925"></a><a name="Deleted Documents"></a>
 <h3 class="boxed">Deleted Documents</h3>
 <p>The .del file is
                     optional, and only exists when a segment contains deletions.
@@ -2571,7 +2575,7 @@ document.write("Last Published: " + docu
 </div>
 
         
-<a name="N10956"></a><a name="Limitations"></a>
+<a name="N1095F"></a><a name="Limitations"></a>
 <h2 class="boxed">Limitations</h2>
 <div class="section">
 <p>

Modified: lucene/dev/branches/positions/lucene/src/site/build/site/gettingstarted.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/lucene/src/site/build/site/gettingstarted.html?rev=1147873&r1=1147872&r2=1147873&view=diff
==============================================================================
--- lucene/dev/branches/positions/lucene/src/site/build/site/gettingstarted.html (original)
+++ lucene/dev/branches/positions/lucene/src/site/build/site/gettingstarted.html Mon Jul 18 13:34:29 2011
@@ -168,9 +168,6 @@ document.write("Last Published: " + docu
 <a href="api/contrib-spellchecker/index.html">Spellchecker</a>
 </div>
 <div class="menuitem">
-<a href="api/contrib-wordnet/index.html">Wordnet</a>
-</div>
-<div class="menuitem">
 <a href="api/contrib-xml-query-parser/index.html">XML Query Parser</a>
 </div>
 </div>

Modified: lucene/dev/branches/positions/lucene/src/site/build/site/index.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/lucene/src/site/build/site/index.html?rev=1147873&r1=1147872&r2=1147873&view=diff
==============================================================================
--- lucene/dev/branches/positions/lucene/src/site/build/site/index.html (original)
+++ lucene/dev/branches/positions/lucene/src/site/build/site/index.html Mon Jul 18 13:34:29 2011
@@ -166,9 +166,6 @@ document.write("Last Published: " + docu
 <a href="api/contrib-spellchecker/index.html">Spellchecker</a>
 </div>
 <div class="menuitem">
-<a href="api/contrib-wordnet/index.html">Wordnet</a>
-</div>
-<div class="menuitem">
 <a href="api/contrib-xml-query-parser/index.html">XML Query Parser</a>
 </div>
 </div>

Modified: lucene/dev/branches/positions/lucene/src/site/build/site/linkmap.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/lucene/src/site/build/site/linkmap.html?rev=1147873&r1=1147872&r2=1147873&view=diff
==============================================================================
--- lucene/dev/branches/positions/lucene/src/site/build/site/linkmap.html (original)
+++ lucene/dev/branches/positions/lucene/src/site/build/site/linkmap.html Mon Jul 18 13:34:29 2011
@@ -166,9 +166,6 @@ document.write("Last Published: " + docu
 <a href="api/contrib-spellchecker/index.html">Spellchecker</a>
 </div>
 <div class="menuitem">
-<a href="api/contrib-wordnet/index.html">Wordnet</a>
-</div>
-<div class="menuitem">
 <a href="api/contrib-xml-query-parser/index.html">XML Query Parser</a>
 </div>
 </div>
@@ -358,12 +355,6 @@ document.write("Last Published: " + docu
 <a href="api/contrib-spellchecker/index.html">Spellchecker</a>&nbsp;&nbsp;___________________&nbsp;&nbsp;<em>javadoc-contrib-spellchecker</em>
 </li>
 </ul>						    
-			
-<ul>
-<li>
-<a href="api/contrib-wordnet/index.html">Wordnet</a>&nbsp;&nbsp;___________________&nbsp;&nbsp;<em>javadoc-contrib-wordnet</em>
-</li>
-</ul>			
 		    
 <ul>
 <li>

Modified: lucene/dev/branches/positions/lucene/src/site/build/site/lucene-contrib/index.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/lucene/src/site/build/site/lucene-contrib/index.html?rev=1147873&r1=1147872&r2=1147873&view=diff
==============================================================================
--- lucene/dev/branches/positions/lucene/src/site/build/site/lucene-contrib/index.html (original)
+++ lucene/dev/branches/positions/lucene/src/site/build/site/lucene-contrib/index.html Mon Jul 18 13:34:29 2011
@@ -168,9 +168,6 @@ document.write("Last Published: " + docu
 <a href="../api/contrib-spellchecker/index.html">Spellchecker</a>
 </div>
 <div class="menuitem">
-<a href="../api/contrib-wordnet/index.html">Wordnet</a>
-</div>
-<div class="menuitem">
 <a href="../api/contrib-xml-query-parser/index.html">XML Query Parser</a>
 </div>
 </div>
@@ -263,9 +260,6 @@ document.write("Last Published: " + docu
 <a href="#spellchecker">spellchecker</a>
 </li>
 <li>
-<a href="#wordnet">wordnet</a>
-</li>
-<li>
 <a href="#xml-query-parser">xml-query-parser</a>
 </li>
 </ul>
@@ -375,15 +369,10 @@ document.write("Last Published: " + docu
 <p>Provides tools for spellchecking and suggestions with Lucene.</p>
 <p>See <a href="../api/contrib-spellchecker/index.html">spellchecker javadoc</a>
 </p>
-<a name="N100DE"></a><a name="wordnet"></a>
-<h3 class="boxed">wordnet</h3>
-<p>Tools to help utilize wordnet synonyms with Lucene</p>
-<p>See <a href="../api/contrib-wordnet/index.html">wordnet javadoc</a>
-</p>
-<a name="N100ED"></a><a name="xml-query-parser"></a>
+<a name="N100DE"></a><a name="xml-query-parser"></a>
 <h3 class="boxed">xml-query-parser</h3>
 <p>A QueryParser that can read queries written in an XML format.</p>
-<p>See <a href="../api/contrib-wordnet/index.html">xml-query-parser javadoc</a>
+<p>See <a href="../api/contrib-xml-query-parser/index.html">xml-query-parser javadoc</a>
 </p>
 </div>
     

Modified: lucene/dev/branches/positions/lucene/src/site/build/site/queryparsersyntax.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/lucene/src/site/build/site/queryparsersyntax.html?rev=1147873&r1=1147872&r2=1147873&view=diff
==============================================================================
--- lucene/dev/branches/positions/lucene/src/site/build/site/queryparsersyntax.html (original)
+++ lucene/dev/branches/positions/lucene/src/site/build/site/queryparsersyntax.html Mon Jul 18 13:34:29 2011
@@ -168,9 +168,6 @@ document.write("Last Published: " + docu
 <a href="api/contrib-spellchecker/index.html">Spellchecker</a>
 </div>
 <div class="menuitem">
-<a href="api/contrib-wordnet/index.html">Wordnet</a>
-</div>
-<div class="menuitem">
 <a href="api/contrib-xml-query-parser/index.html">XML Query Parser</a>
 </div>
 </div>

Modified: lucene/dev/branches/positions/lucene/src/site/build/site/scoring.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/lucene/src/site/build/site/scoring.html?rev=1147873&r1=1147872&r2=1147873&view=diff
==============================================================================
--- lucene/dev/branches/positions/lucene/src/site/build/site/scoring.html (original)
+++ lucene/dev/branches/positions/lucene/src/site/build/site/scoring.html Mon Jul 18 13:34:29 2011
@@ -168,9 +168,6 @@ document.write("Last Published: " + docu
 <a href="api/contrib-spellchecker/index.html">Spellchecker</a>
 </div>
 <div class="menuitem">
-<a href="api/contrib-wordnet/index.html">Wordnet</a>
-</div>
-<div class="menuitem">
 <a href="api/contrib-xml-query-parser/index.html">XML Query Parser</a>
 </div>
 </div>

Modified: lucene/dev/branches/positions/lucene/src/site/build/site/systemrequirements.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/lucene/src/site/build/site/systemrequirements.html?rev=1147873&r1=1147872&r2=1147873&view=diff
==============================================================================
--- lucene/dev/branches/positions/lucene/src/site/build/site/systemrequirements.html (original)
+++ lucene/dev/branches/positions/lucene/src/site/build/site/systemrequirements.html Mon Jul 18 13:34:29 2011
@@ -166,9 +166,6 @@ document.write("Last Published: " + docu
 <a href="api/contrib-spellchecker/index.html">Spellchecker</a>
 </div>
 <div class="menuitem">
-<a href="api/contrib-wordnet/index.html">Wordnet</a>
-</div>
-<div class="menuitem">
 <a href="api/contrib-xml-query-parser/index.html">XML Query Parser</a>
 </div>
 </div>

Modified: lucene/dev/branches/positions/lucene/src/site/src/documentation/content/xdocs/fileformats.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/lucene/src/site/src/documentation/content/xdocs/fileformats.xml?rev=1147873&r1=1147872&r2=1147873&view=diff
==============================================================================
--- lucene/dev/branches/positions/lucene/src/site/src/documentation/content/xdocs/fileformats.xml (original)
+++ lucene/dev/branches/positions/lucene/src/site/src/documentation/content/xdocs/fileformats.xml Mon Jul 18 13:34:29 2011
@@ -99,6 +99,10 @@
             to stored fields file, previously they were stored in
             text format only.
            </p>
+        <p>
+            In version 3.4, fields can omit position data while
+            still indexing term frequencies.
+        </p>
         </section>
 
         <section id="Definitions"><title>Definitions</title>
@@ -276,7 +280,7 @@
                     <p>Term Frequency
                         data. For each term in the dictionary, the numbers of all the
                         documents that contain that term, and the frequency of the term in
-                        that document if omitTf is false.
+                        that document, unless frequencies are omitted (IndexOptions.DOCS_ONLY)
                     </p>
                 </li>
 
@@ -284,8 +288,7 @@
                     <p>Term Proximity
                         data. For each term in the dictionary, the positions that the term
                         occurs in each document.  Note that this will
-                        not exist if all fields in all documents set
-                        omitTf to true.
+                        not exist if all fields in all documents omit position data.
                     </p>
                 </li>
 
@@ -1080,7 +1083,7 @@
 
 		<p>
 		    HasProx is 1 if any fields in this segment have
-		    omitTf set to false; else, it's 0.
+		    position data (IndexOptions.DOCS_AND_FREQS_AND_POSITIONS); else, it's 0.
 		</p>
 
 		<p>
@@ -1217,11 +1220,13 @@
                         <li>If the fourth lowest-order bit is set (0x08), term offsets are stored with the term vectors.</li>
                         <li>If the fifth lowest-order bit is set (0x10), norms are omitted for the indexed field.</li>
                         <li>If the sixth lowest-order bit is set (0x20), payloads are stored for the indexed field.</li>
+                        <li>If the seventh lowest-order bit is set (0x40), term frequencies and positions omitted for the indexed field.</li>
+                        <li>If the eighth lowest-order bit is set (0x80), positions are omitted for the indexed field.</li>
                     </ul>
                 </p>
 
 		<p>
-		   FNMVersion (added in 2.9) is always -2.
+		   FNMVersion (added in 2.9) is -2 for indexes from 2.9 - 3.3. It is -3 for indexes in Lucene 3.4+
 		</p>
 
                 <p>
@@ -1419,7 +1424,7 @@
                             file. In particular, it is the difference between the position of
                             this term's data in that file and the position of the previous
                             term's data (or zero, for the first term in the file.  For fields
-			    with omitTf true, this will be 0 since
+			                that omit position data, this will be 0 since
                             prox information is not stored.
                         </p>
                         <p>SkipDelta determines the position of this
@@ -1494,7 +1499,7 @@
                 <p>
                     The .frq file contains the lists of documents
                     which contain each term, along with the frequency of the term in that
-                    document (if omitTf is false).
+                    document (except when frequencies are omitted: IndexOptions.DOCS_ONLY).
                 </p>
                 <p>FreqFile (.frq) --&gt;
                     &lt;TermFreqs, SkipData&gt;
@@ -1531,26 +1536,26 @@
                 <p>TermFreq
                     entries are ordered by increasing document number.
                 </p>
-                <p>DocDelta: if omitTf is false, this determines both
+                <p>DocDelta: if frequencies are indexed, this determines both
                     the document number and the frequency. In
                     particular, DocDelta/2 is the difference between
                     this document number and the previous document
                     number (or zero when this is the first document in
                     a TermFreqs). When DocDelta is odd, the frequency
                     is one. When DocDelta is even, the frequency is
-                    read as another VInt.  If omitTf is true, DocDelta
+                    read as another VInt.  If frequencies are omitted, DocDelta
                     contains the gap (not multiplied by 2) between
                     document numbers and no frequency information is
                     stored.
                 </p>
                 <p>For example, the TermFreqs for a term which occurs
                     once in document seven and three times in document
-                    eleven, with omitTf false, would be the following
+                    eleven, with frequencies indexed, would be the following
                     sequence of VInts:
                 </p>
                 <p>15, 8, 3
                 </p>
-		<p> If omitTf were true it would be this sequence
+		<p> If frequencies were omitted (IndexOptions.DOCS_ONLY) it would be this sequence
 		of VInts instead:
 		  </p>
 		 <p>
@@ -1621,9 +1626,9 @@
                 <p>
                     The .prx file contains the lists of positions that
                     each term occurs at within documents.  Note that
-                    fields with omitTf true do not store
+                    fields omitting positional data do not store
                     anything into this file, and if all fields in the
-                    index have omitTf true then the .prx file will not
+                    index omit positional data then the .prx file will not
                     exist.
                 </p>
                 <p>ProxFile (.prx) --&gt;

Modified: lucene/dev/branches/positions/lucene/src/site/src/documentation/content/xdocs/lucene-contrib/index.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/lucene/src/site/src/documentation/content/xdocs/lucene-contrib/index.xml?rev=1147873&r1=1147872&r2=1147873&view=diff
==============================================================================
--- lucene/dev/branches/positions/lucene/src/site/src/documentation/content/xdocs/lucene-contrib/index.xml (original)
+++ lucene/dev/branches/positions/lucene/src/site/src/documentation/content/xdocs/lucene-contrib/index.xml Mon Jul 18 13:34:29 2011
@@ -106,14 +106,9 @@
                 <p>See <a href="../api/contrib-spellchecker/index.html">spellchecker javadoc</a></p>
             </section>                     
             
-            <section id="wordnet"><title>wordnet</title>
-                <p>Tools to help utilize wordnet synonyms with Lucene</p>
-                <p>See <a href="../api/contrib-wordnet/index.html">wordnet javadoc</a></p>
-            </section>  
-            
             <section id="xml-query-parser"><title>xml-query-parser</title>
                 <p>A QueryParser that can read queries written in an XML format.</p>
-                <p>See <a href="../api/contrib-wordnet/index.html">xml-query-parser javadoc</a></p>
+                <p>See <a href="../api/contrib-xml-query-parser/index.html">xml-query-parser javadoc</a></p>
             </section>  
            
 		</section>

Modified: lucene/dev/branches/positions/lucene/src/site/src/documentation/content/xdocs/site.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/lucene/src/site/src/documentation/content/xdocs/site.xml?rev=1147873&r1=1147872&r2=1147873&view=diff
==============================================================================
--- lucene/dev/branches/positions/lucene/src/site/src/documentation/content/xdocs/site.xml (original)
+++ lucene/dev/branches/positions/lucene/src/site/src/documentation/content/xdocs/site.xml Mon Jul 18 13:34:29 2011
@@ -66,7 +66,6 @@ See http://forrest.apache.org/docs/linki
 		    <javadoc-contrib-remote label="Remote" href="ext:javadocs-contrib-remote"/>
 		    <javadoc-contrib-spatial label="Spatial" href="ext:javadocs-contrib-spatial"/>
 		    <javadoc-contrib-spellchecker label="Spellchecker" href="ext:javadocs-contrib-spellchecker"/>						    
-			<javadoc-contrib-wordnet label="Wordnet" href="ext:javadocs-contrib-wordnet"/>			
 		    <javadoc-contrib-xml-query-parser label="XML Query Parser" href="ext:javadocs-contrib-xml-query-parser"/>			
 		 </javadoc-contrib>
 	  </javadoc>
@@ -106,7 +105,6 @@ See http://forrest.apache.org/docs/linki
 	<javadocs-contrib-remote href="api/contrib-remote/index.html"/>
 	<javadocs-contrib-spatial href="api/contrib-spatial/index.html"/>
 	<javadocs-contrib-spellchecker href="api/contrib-spellchecker/index.html"/>
-	<javadocs-contrib-wordnet href="api/contrib-wordnet/index.html"/>
 	<javadocs-contrib-xml-query-parser href="api/contrib-xml-query-parser/index.html"/>
 	
     <forrest href="http://forrest.apache.org/">

Modified: lucene/dev/branches/positions/lucene/src/test-framework/org/apache/lucene/analysis/BaseTokenStreamTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/lucene/src/test-framework/org/apache/lucene/analysis/BaseTokenStreamTestCase.java?rev=1147873&r1=1147872&r2=1147873&view=diff
==============================================================================
--- lucene/dev/branches/positions/lucene/src/test-framework/org/apache/lucene/analysis/BaseTokenStreamTestCase.java (original)
+++ lucene/dev/branches/positions/lucene/src/test-framework/org/apache/lucene/analysis/BaseTokenStreamTestCase.java Mon Jul 18 13:34:29 2011
@@ -260,7 +260,11 @@ public abstract class BaseTokenStreamTes
         default:
           text = _TestUtil.randomUnicodeString(random, maxWordLength);
       }
-      
+
+      if (VERBOSE) {
+        System.out.println("NOTE: BaseTokenStreamTestCase: get first token stream now text=" + text);
+      }
+
       TokenStream ts = a.reusableTokenStream("dummy", new StringReader(text));
       assertTrue("has no CharTermAttribute", ts.hasAttribute(CharTermAttribute.class));
       CharTermAttribute termAtt = ts.getAttribute(CharTermAttribute.class);
@@ -286,6 +290,9 @@ public abstract class BaseTokenStreamTes
       ts.close();
       // verify reusing is "reproducable" and also get the normal tokenstream sanity checks
       if (!tokens.isEmpty()) {
+        if (VERBOSE) {
+          System.out.println("NOTE: BaseTokenStreamTestCase: re-run analysis");
+        }
         if (typeAtt != null && posIncAtt != null && offsetAtt != null) {
           // offset + pos + type
           assertAnalyzesToReuse(a, text, 

Modified: lucene/dev/branches/positions/lucene/src/test-framework/org/apache/lucene/index/DocHelper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/lucene/src/test-framework/org/apache/lucene/index/DocHelper.java?rev=1147873&r1=1147872&r2=1147873&view=diff
==============================================================================
--- lucene/dev/branches/positions/lucene/src/test-framework/org/apache/lucene/index/DocHelper.java (original)
+++ lucene/dev/branches/positions/lucene/src/test-framework/org/apache/lucene/index/DocHelper.java Mon Jul 18 13:34:29 2011
@@ -29,6 +29,7 @@ import org.apache.lucene.analysis.MockTo
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.Fieldable;
+import org.apache.lucene.index.FieldInfo.IndexOptions;
 import org.apache.lucene.search.SimilarityProvider;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.util.LuceneTestCase;
@@ -67,7 +68,7 @@ class DocHelper {
   public static Field noTFField = new Field(NO_TF_KEY, NO_TF_TEXT,
       Field.Store.YES, Field.Index.ANALYZED);
   static {
-    noTFField.setOmitTermFreqAndPositions(true);
+    noTFField.setIndexOptions(IndexOptions.DOCS_ONLY);
   }
 
   public static final String UNINDEXED_FIELD_TEXT = "unindexed field text";
@@ -173,7 +174,7 @@ class DocHelper {
       if (f.isStored()) add(stored,f);
       else add(unstored,f);
       if (f.getOmitNorms()) add(noNorms,f);
-      if (f.getOmitTermFreqAndPositions()) add(noTf,f);
+      if (f.getIndexOptions() == IndexOptions.DOCS_ONLY) add(noTf,f);
       if (f.isLazy()) add(lazy, f);
     }
   }

Modified: lucene/dev/branches/positions/lucene/src/test-framework/org/apache/lucene/index/codecs/preflexrw/PreFlexFieldsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/lucene/src/test-framework/org/apache/lucene/index/codecs/preflexrw/PreFlexFieldsWriter.java?rev=1147873&r1=1147872&r2=1147873&view=diff
==============================================================================
--- lucene/dev/branches/positions/lucene/src/test-framework/org/apache/lucene/index/codecs/preflexrw/PreFlexFieldsWriter.java (original)
+++ lucene/dev/branches/positions/lucene/src/test-framework/org/apache/lucene/index/codecs/preflexrw/PreFlexFieldsWriter.java Mon Jul 18 13:34:29 2011
@@ -22,6 +22,7 @@ import java.util.Comparator;
 
 import org.apache.lucene.index.CorruptIndexException;
 import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.FieldInfo.IndexOptions;
 import org.apache.lucene.index.IndexFileNames;
 import org.apache.lucene.index.SegmentWriteState;
 import org.apache.lucene.index.codecs.FieldsConsumer;
@@ -90,7 +91,7 @@ class PreFlexFieldsWriter extends Fields
 
     public PreFlexTermsWriter(FieldInfo fieldInfo) {
       this.fieldInfo = fieldInfo;
-      omitTF = fieldInfo.omitTermFreqAndPositions;
+      omitTF = fieldInfo.indexOptions == IndexOptions.DOCS_ONLY;
       storePayloads = fieldInfo.storePayloads;
     }
 
@@ -195,7 +196,7 @@ class PreFlexFieldsWriter extends Fields
     }
 
     @Override
-    public void finish(long sumTotalTermCount) throws IOException {
+    public void finish(long sumTotalTermCount, long sumDocFreq) throws IOException {
     }
 
     @Override

Modified: lucene/dev/branches/positions/lucene/src/test-framework/org/apache/lucene/util/LuceneTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/lucene/src/test-framework/org/apache/lucene/util/LuceneTestCase.java?rev=1147873&r1=1147872&r2=1147873&view=diff
==============================================================================
--- lucene/dev/branches/positions/lucene/src/test-framework/org/apache/lucene/util/LuceneTestCase.java (original)
+++ lucene/dev/branches/positions/lucene/src/test-framework/org/apache/lucene/util/LuceneTestCase.java Mon Jul 18 13:34:29 2011
@@ -352,6 +352,7 @@ public abstract class LuceneTestCase ext
     state = State.INITIAL;
     staticSeed = "random".equals(TEST_SEED) ? seedRand.nextLong() : TwoLongs.fromString(TEST_SEED).l1;
     random.setSeed(staticSeed);
+    random.initialized = true;
     tempDirs.clear();
     stores = Collections.synchronizedMap(new IdentityHashMap<MockDirectoryWrapper,StackTraceElement[]>());
     
@@ -494,6 +495,8 @@ public abstract class LuceneTestCase ext
         }
       }
     }
+    random.setSeed(0L);
+    random.initialized = false;
   }
 
   private static boolean testsFailed; /* true if any tests failed */
@@ -1366,7 +1369,26 @@ public abstract class LuceneTestCase ext
   private long seed;
 
   private static final Random seedRand = new Random();
-  protected static final Random random = new Random(0);
+  protected static final SmartRandom random = new SmartRandom(0);
+  
+  public static class SmartRandom extends Random {
+    boolean initialized;
+    
+    SmartRandom(long seed) {
+      super(seed);
+    }
+    
+    @Override
+    protected int next(int bits) {
+      if (!initialized) {
+        System.err.println("!!! WARNING: test is using random from static initializer !!!");
+        Thread.dumpStack();
+        // I wish, but it causes JRE crashes
+        // throw new IllegalStateException("you cannot use this random from a static initializer in your test");
+      }
+      return super.next(bits);
+    }
+  }
 
   private String name = "<unknown>";
 

Modified: lucene/dev/branches/positions/lucene/src/test-framework/org/apache/lucene/util/_TestUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/lucene/src/test-framework/org/apache/lucene/util/_TestUtil.java?rev=1147873&r1=1147872&r2=1147873&view=diff
==============================================================================
--- lucene/dev/branches/positions/lucene/src/test-framework/org/apache/lucene/util/_TestUtil.java (original)
+++ lucene/dev/branches/positions/lucene/src/test-framework/org/apache/lucene/util/_TestUtil.java Mon Jul 18 13:34:29 2011
@@ -422,7 +422,7 @@ public class _TestUtil {
     List<Fieldable> fields = doc.getFields();
     for (Fieldable field : fields) {
       fieldInfos.addOrUpdate(field.name(), field.isIndexed(), field.isTermVectorStored(), field.isStorePositionWithTermVector(),
-              field.isStoreOffsetWithTermVector(), field.getOmitNorms(), false, field.getOmitTermFreqAndPositions(), field.docValuesType());
+              field.isStoreOffsetWithTermVector(), field.getOmitNorms(), false, field.getIndexOptions(), field.docValuesType());
     }
   }
   
@@ -507,7 +507,7 @@ public class _TestUtil {
                                field1.isStored() ? Field.Store.YES : Field.Store.NO,
                                field1.isIndexed() ? (field1.isTokenized() ? Field.Index.ANALYZED : Field.Index.NOT_ANALYZED) : Field.Index.NO);
       field2.setOmitNorms(field1.getOmitNorms());
-      field2.setOmitTermFreqAndPositions(field1.getOmitTermFreqAndPositions());
+      field2.setIndexOptions(field1.getIndexOptions());
       doc2.add(field2);
     }
 

Modified: lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/TestExternalCodecs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/TestExternalCodecs.java?rev=1147873&r1=1147872&r2=1147873&view=diff
==============================================================================
--- lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/TestExternalCodecs.java (original)
+++ lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/TestExternalCodecs.java Mon Jul 18 13:34:29 2011
@@ -102,6 +102,7 @@ public class TestExternalCodecs extends 
       final String field;
       final SortedMap<String,RAMTerm> termToDocs = new TreeMap<String,RAMTerm>();
       long sumTotalTermFreq;
+      long sumDocFreq;
 
       RAMField(String field) {
         this.field = field;
@@ -116,6 +117,11 @@ public class TestExternalCodecs extends 
       public long getSumTotalTermFreq() {
         return sumTotalTermFreq;
       }
+      
+      @Override
+      public long getSumDocFreq() throws IOException {
+        return sumDocFreq;
+      }
 
       @Override
       public TermsEnum iterator() {
@@ -204,8 +210,9 @@ public class TestExternalCodecs extends 
       }
 
       @Override
-      public void finish(long sumTotalTermFreq) {
+      public void finish(long sumTotalTermFreq, long sumDocFreq) {
         field.sumTotalTermFreq = sumTotalTermFreq;
+        field.sumDocFreq = sumDocFreq;
       }
     }
 

Modified: lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/Test2BTerms.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/Test2BTerms.java?rev=1147873&r1=1147872&r2=1147873&view=diff
==============================================================================
--- lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/Test2BTerms.java (original)
+++ lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/Test2BTerms.java Mon Jul 18 13:34:29 2011
@@ -23,6 +23,7 @@ import org.apache.lucene.search.*;
 import org.apache.lucene.analysis.*;
 import org.apache.lucene.analysis.tokenattributes.*;
 import org.apache.lucene.document.*;
+import org.apache.lucene.index.FieldInfo.IndexOptions;
 import org.apache.lucene.index.codecs.CodecProvider;
 import java.io.File;
 import java.io.IOException;
@@ -177,7 +178,7 @@ public class Test2BTerms extends LuceneT
       Document doc = new Document();
       final MyTokenStream ts = new MyTokenStream(random, TERMS_PER_DOC);
       Field field = new Field("field", ts);
-      field.setOmitTermFreqAndPositions(true);
+      field.setIndexOptions(IndexOptions.DOCS_ONLY);
       field.setOmitNorms(true);
       doc.add(field);
       //w.setInfoStream(System.out);

Modified: lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java?rev=1147873&r1=1147872&r2=1147873&view=diff
==============================================================================
--- lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java (original)
+++ lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java Mon Jul 18 13:34:29 2011
@@ -31,6 +31,7 @@ import org.apache.lucene.document.Docume
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.Fieldable;
 import org.apache.lucene.document.NumericField;
+import org.apache.lucene.index.FieldInfo.IndexOptions;
 import org.apache.lucene.index.IndexWriterConfig.OpenMode;
 import org.apache.lucene.search.DefaultSimilarity;
 import org.apache.lucene.search.DocIdSetIterator;
@@ -95,6 +96,8 @@ public class TestBackwardsCompatibility 
                              "31.nocfs",
                              "32.cfs",
                              "32.nocfs",
+                             "34.cfs",
+                             "34.nocfs",
   };
   
   final String[] unsupportedNames = {"19.cfs",
@@ -455,9 +458,12 @@ public class TestBackwardsCompatibility 
     File indexDir = new File(LuceneTestCase.TEMP_DIR, dirName);
     _TestUtil.rmDir(indexDir);
     Directory dir = newFSDirectory(indexDir);
-    
-    IndexWriterConfig conf = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMaxBufferedDocs(10);
-    ((LogMergePolicy) conf.getMergePolicy()).setUseCompoundFile(doCFS);
+    LogByteSizeMergePolicy mp = new LogByteSizeMergePolicy();
+    mp.setUseCompoundFile(doCFS);
+    mp.setNoCFSRatio(1.0);
+    // TODO: remove randomness
+    IndexWriterConfig conf = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))
+      .setMaxBufferedDocs(10).setMergePolicy(mp);
     IndexWriter writer = new IndexWriter(dir, conf);
     
     for(int i=0;i<35;i++) {
@@ -471,8 +477,12 @@ public class TestBackwardsCompatibility 
 
     if (!optimized) {
       // open fresh writer so we get no prx file in the added segment
-      conf = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMaxBufferedDocs(10);
-      ((LogMergePolicy) conf.getMergePolicy()).setUseCompoundFile(doCFS);
+      mp = new LogByteSizeMergePolicy();
+      mp.setUseCompoundFile(doCFS);
+      mp.setNoCFSRatio(1.0);
+      // TODO: remove randomness
+      conf = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))
+        .setMaxBufferedDocs(10).setMergePolicy(mp);
       writer = new IndexWriter(dir, conf);
       addNoProxDoc(writer);
       writer.close();
@@ -599,10 +609,10 @@ public class TestBackwardsCompatibility 
   private void addNoProxDoc(IndexWriter writer) throws IOException {
     Document doc = new Document();
     Field f = new Field("content3", "aaa", Field.Store.YES, Field.Index.ANALYZED);
-    f.setOmitTermFreqAndPositions(true);
+    f.setIndexOptions(IndexOptions.DOCS_ONLY);
     doc.add(f);
     f = new Field("content4", "aaa", Field.Store.YES, Field.Index.NO);
-    f.setOmitTermFreqAndPositions(true);
+    f.setIndexOptions(IndexOptions.DOCS_ONLY);
     doc.add(f);
     writer.addDocument(doc);
   }

Modified: lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/TestCodecs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/TestCodecs.java?rev=1147873&r1=1147872&r2=1147873&view=diff
==============================================================================
--- lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/TestCodecs.java (original)
+++ lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/TestCodecs.java Mon Jul 18 13:34:29 2011
@@ -25,6 +25,7 @@ import org.apache.lucene.analysis.MockAn
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.Field.Store;
+import org.apache.lucene.index.FieldInfo.IndexOptions;
 import org.apache.lucene.index.codecs.CodecProvider;
 import org.apache.lucene.index.codecs.FieldsConsumer;
 import org.apache.lucene.index.codecs.FieldsProducer;
@@ -45,6 +46,7 @@ import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.util.Version;
 import org.apache.lucene.util._TestUtil;
+import org.junit.BeforeClass;
 
 // TODO: test multiple codecs here?
 
@@ -66,13 +68,18 @@ import org.apache.lucene.util._TestUtil;
 public class TestCodecs extends LuceneTestCase {
   private static String[] fieldNames = new String[] {"one", "two", "three", "four"};
 
-  private final static int NUM_TEST_ITER = atLeast(20);
+  private static int NUM_TEST_ITER;
   private final static int NUM_TEST_THREADS = 3;
   private final static int NUM_FIELDS = 4;
   private final static int NUM_TERMS_RAND = 50; // must be > 16 to test skipping
   private final static int DOC_FREQ_RAND = 500; // must be > 16 to test skipping
   private final static int TERM_DOC_FREQ_RAND = 20;
 
+  @BeforeClass
+  public static void beforeClass() throws Exception {
+    NUM_TEST_ITER = atLeast(20);
+  }
+
   class FieldData implements Comparable {
     final FieldInfo fieldInfo;
     final TermData[] terms;
@@ -84,7 +91,8 @@ public class TestCodecs extends LuceneTe
       this.storePayloads = storePayloads;
       fieldInfos.addOrUpdate(name, true);
       fieldInfo = fieldInfos.fieldInfo(name);
-      fieldInfo.omitTermFreqAndPositions = omitTF;
+      // TODO: change this test to use all three
+      fieldInfo.indexOptions = omitTF ? IndexOptions.DOCS_ONLY : IndexOptions.DOCS_AND_FREQS_AND_POSITIONS;
       fieldInfo.storePayloads = storePayloads;
       this.terms = terms;
       for(int i=0;i<terms.length;i++)
@@ -101,10 +109,12 @@ public class TestCodecs extends LuceneTe
       Arrays.sort(terms);
       final TermsConsumer termsConsumer = consumer.addField(fieldInfo);
       long sumTotalTermCount = 0;
+      long sumDF = 0;
       for (final TermData term : terms) {
+        sumDF += term.docs.length;
         sumTotalTermCount += term.write(termsConsumer);
       }
-      termsConsumer.finish(sumTotalTermCount);
+      termsConsumer.finish(sumTotalTermCount, sumDF);
     }
   }
 

Modified: lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/TestDocumentWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/TestDocumentWriter.java?rev=1147873&r1=1147872&r2=1147873&view=diff
==============================================================================
--- lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/TestDocumentWriter.java (original)
+++ lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/TestDocumentWriter.java Mon Jul 18 13:34:29 2011
@@ -34,6 +34,7 @@ import org.apache.lucene.document.Field.
 import org.apache.lucene.document.Field.Store;
 import org.apache.lucene.document.Field.TermVector;
 import org.apache.lucene.document.Fieldable;
+import org.apache.lucene.index.FieldInfo.IndexOptions;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.IOContext;
 import org.apache.lucene.store.IOContext.Context;
@@ -303,7 +304,7 @@ public class TestDocumentWriter extends 
     doc.add(newField("f1", "v2", Store.YES, Index.NO));
     // f2 has no TF
     Field f = newField("f2", "v1", Store.NO, Index.ANALYZED);
-    f.setOmitTermFreqAndPositions(true);
+    f.setIndexOptions(IndexOptions.DOCS_ONLY);
     doc.add(f);
     doc.add(newField("f2", "v2", Store.YES, Index.NO));
 
@@ -319,10 +320,10 @@ public class TestDocumentWriter extends 
     FieldInfos fi = reader.fieldInfos();
     // f1
     assertFalse("f1 should have no norms", reader.hasNorms("f1"));
-    assertFalse("omitTermFreqAndPositions field bit should not be set for f1", fi.fieldInfo("f1").omitTermFreqAndPositions);
+    assertEquals("omitTermFreqAndPositions field bit should not be set for f1", IndexOptions.DOCS_AND_FREQS_AND_POSITIONS, fi.fieldInfo("f1").indexOptions);
     // f2
     assertTrue("f2 should have norms", reader.hasNorms("f2"));
-    assertTrue("omitTermFreqAndPositions field bit should be set for f2", fi.fieldInfo("f2").omitTermFreqAndPositions);
+    assertEquals("omitTermFreqAndPositions field bit should be set for f2", IndexOptions.DOCS_ONLY, fi.fieldInfo("f2").indexOptions);
     reader.close();
   }
 }

Modified: lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/TestFieldInfos.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/TestFieldInfos.java?rev=1147873&r1=1147872&r2=1147873&view=diff
==============================================================================
--- lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/TestFieldInfos.java (original)
+++ lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/TestFieldInfos.java Mon Jul 18 13:34:29 2011
@@ -20,6 +20,7 @@ package org.apache.lucene.index;
 import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.util._TestUtil;
 import org.apache.lucene.document.Document;
+import org.apache.lucene.index.FieldInfo.IndexOptions;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.IndexOutput;
 
@@ -137,7 +138,7 @@ public class TestFieldInfos extends Luce
     try {
       readOnly.addOrUpdate("bogus", random.nextBoolean(), random.nextBoolean(),
           random.nextBoolean(), random.nextBoolean(), random.nextBoolean(),
-          random.nextBoolean(), random.nextBoolean(), null);
+          random.nextBoolean(), random.nextBoolean() ? IndexOptions.DOCS_ONLY : IndexOptions.DOCS_AND_FREQS_AND_POSITIONS, null);
       fail("instance should be read only");
     } catch (IllegalStateException e) {
       // expected

Modified: lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/TestFieldsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/TestFieldsReader.java?rev=1147873&r1=1147872&r2=1147873&view=diff
==============================================================================
--- lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/TestFieldsReader.java (original)
+++ lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/TestFieldsReader.java Mon Jul 18 13:34:29 2011
@@ -30,6 +30,7 @@ import org.apache.lucene.document.FieldS
 import org.apache.lucene.document.Fieldable;
 import org.apache.lucene.document.LoadFirstFieldSelector;
 import org.apache.lucene.document.SetBasedFieldSelector;
+import org.apache.lucene.index.FieldInfo.IndexOptions;
 import org.apache.lucene.index.IndexWriterConfig.OpenMode;
 import org.apache.lucene.search.FieldCache;
 import org.apache.lucene.store.AlreadyClosedException;
@@ -91,7 +92,7 @@ public class TestFieldsReader extends Lu
     assertTrue(field.isStoreOffsetWithTermVector() == true);
     assertTrue(field.isStorePositionWithTermVector() == true);
     assertTrue(field.getOmitNorms() == false);
-    assertTrue(field.getOmitTermFreqAndPositions() == false);
+    assertTrue(field.getIndexOptions() == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
 
     field = doc.getField(DocHelper.TEXT_FIELD_3_KEY);
     assertTrue(field != null);
@@ -99,7 +100,7 @@ public class TestFieldsReader extends Lu
     assertTrue(field.isStoreOffsetWithTermVector() == false);
     assertTrue(field.isStorePositionWithTermVector() == false);
     assertTrue(field.getOmitNorms() == true);
-    assertTrue(field.getOmitTermFreqAndPositions() == false);
+    assertTrue(field.getIndexOptions() == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
 
     field = doc.getField(DocHelper.NO_TF_KEY);
     assertTrue(field != null);
@@ -107,7 +108,7 @@ public class TestFieldsReader extends Lu
     assertTrue(field.isStoreOffsetWithTermVector() == false);
     assertTrue(field.isStorePositionWithTermVector() == false);
     assertTrue(field.getOmitNorms() == false);
-    assertTrue(field.getOmitTermFreqAndPositions() == true);
+    assertTrue(field.getIndexOptions() == IndexOptions.DOCS_ONLY);
     reader.close();
   }
 

Modified: lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/TestIndexWriterCommit.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/TestIndexWriterCommit.java?rev=1147873&r1=1147872&r2=1147873&view=diff
==============================================================================
--- lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/TestIndexWriterCommit.java (original)
+++ lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/TestIndexWriterCommit.java Mon Jul 18 13:34:29 2011
@@ -31,6 +31,7 @@ import org.apache.lucene.analysis.TokenS
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.index.IndexWriterConfig.OpenMode;
+import org.apache.lucene.index.codecs.CodecProvider;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.ScoreDoc;
 import org.apache.lucene.search.TermQuery;
@@ -166,6 +167,13 @@ public class TestIndexWriterCommit exten
    * measure max temp disk space used.
    */
   public void testCommitOnCloseDiskUsage() throws IOException {
+    // MemoryCodec, since it uses FST, is not necessarily
+    // "additive", ie if you add up N small FSTs, then merge
+    // them, the merged result can easily be larger than the
+    // sum because the merged FST may use array encoding for
+    // some arcs (which uses more space):
+    assumeFalse("This test cannot run with Memory codec", CodecProvider.getDefault().getFieldCodec("id").equals("Memory"));
+    assumeFalse("This test cannot run with Memory codec", CodecProvider.getDefault().getFieldCodec("content").equals("Memory"));
     MockDirectoryWrapper dir = newDirectory();
     Analyzer analyzer;
     if (random.nextBoolean()) {

Modified: lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java?rev=1147873&r1=1147872&r2=1147873&view=diff
==============================================================================
--- lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java (original)
+++ lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java Mon Jul 18 13:34:29 2011
@@ -23,6 +23,7 @@ import org.apache.lucene.analysis.MockAn
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.index.IndexWriterConfig.OpenMode;
+import org.apache.lucene.index.codecs.CodecProvider;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.ScoreDoc;
 import org.apache.lucene.search.TermQuery;
@@ -142,6 +143,14 @@ public class TestIndexWriterOnDiskFull e
    */
   public void testAddIndexOnDiskFull() throws IOException
   {
+    // MemoryCodec, since it uses FST, is not necessarily
+    // "additive", ie if you add up N small FSTs, then merge
+    // them, the merged result can easily be larger than the
+    // sum because the merged FST may use array encoding for
+    // some arcs (which uses more space):
+    assumeFalse("This test cannot run with Memory codec", CodecProvider.getDefault().getFieldCodec("id").equals("Memory"));
+    assumeFalse("This test cannot run with Memory codec", CodecProvider.getDefault().getFieldCodec("content").equals("Memory"));
+
     int START_COUNT = 57;
     int NUM_DIR = TEST_NIGHTLY ? 50 : 5;
     int END_COUNT = START_COUNT + NUM_DIR* (TEST_NIGHTLY ? 25 : 5);

Modified: lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/TestLongPostings.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/TestLongPostings.java?rev=1147873&r1=1147872&r2=1147873&view=diff
==============================================================================
--- lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/TestLongPostings.java (original)
+++ lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/TestLongPostings.java Mon Jul 18 13:34:29 2011
@@ -26,11 +26,12 @@ import org.apache.lucene.analysis.TokenS
 import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
+import org.apache.lucene.index.FieldInfo.IndexOptions;
 import org.apache.lucene.index.codecs.CodecProvider;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.util.OpenBitSet;
+import org.apache.lucene.util.FixedBitSet;
 import org.apache.lucene.util._TestUtil;
 
 public class TestLongPostings extends LuceneTestCase {
@@ -63,8 +64,8 @@ public class TestLongPostings extends Lu
   }
 
   public void testLongPostings() throws Exception {
-    assumeFalse("Too slow with SimpleText codec", CodecProvider.getDefault().getFieldCodec("field").equals("SimpleText"));
-    assumeFalse("Too slow with Memory codec", CodecProvider.getDefault().getFieldCodec("field").equals("Memory"));
+    assumeFalse("Too slow with SimpleText codec at night", TEST_NIGHTLY && CodecProvider.getDefault().getFieldCodec("field").equals("SimpleText"));
+    assumeFalse("Too slow with Memory codec at night", TEST_NIGHTLY && CodecProvider.getDefault().getFieldCodec("field").equals("Memory"));
 
     // Don't use _TestUtil.getTempDir so that we own the
     // randomness (ie same seed will point to same dir):
@@ -91,7 +92,7 @@ public class TestLongPostings extends Lu
       */
     }
 
-    final OpenBitSet isS1 = new OpenBitSet(NUM_DOCS);
+    final FixedBitSet isS1 = new FixedBitSet(NUM_DOCS);
     for(int idx=0;idx<NUM_DOCS;idx++) {
       if (random.nextBoolean()) {
         isS1.set(idx);
@@ -250,4 +251,187 @@ public class TestLongPostings extends Lu
     r.close();
     dir.close();
   }
+  
+  // a weaker form of testLongPostings, that doesnt check positions
+  public void testLongPostingsNoPositions() throws Exception {
+    doTestLongPostingsNoPositions(IndexOptions.DOCS_ONLY);
+    doTestLongPostingsNoPositions(IndexOptions.DOCS_AND_FREQS);
+  }
+  
+  public void doTestLongPostingsNoPositions(IndexOptions options) throws Exception {
+    assumeFalse("Too slow with SimpleText codec at night", TEST_NIGHTLY && CodecProvider.getDefault().getFieldCodec("field").equals("SimpleText"));
+    assumeFalse("Too slow with Memory codec at night", TEST_NIGHTLY && CodecProvider.getDefault().getFieldCodec("field").equals("Memory"));
+    // Don't use _TestUtil.getTempDir so that we own the
+    // randomness (ie same seed will point to same dir):
+    Directory dir = newFSDirectory(_TestUtil.getTempDir("longpostings" + "." + random.nextLong()));
+
+    final int NUM_DOCS = atLeast(2000);
+
+    if (VERBOSE) {
+      System.out.println("TEST: NUM_DOCS=" + NUM_DOCS);
+    }
+
+    final String s1 = getRandomTerm(null);
+    final String s2 = getRandomTerm(s1);
+
+    if (VERBOSE) {
+      System.out.println("\nTEST: s1=" + s1 + " s2=" + s2);
+      /*
+      for(int idx=0;idx<s1.length();idx++) {
+        System.out.println("  s1 ch=0x" + Integer.toHexString(s1.charAt(idx)));
+      }
+      for(int idx=0;idx<s2.length();idx++) {
+        System.out.println("  s2 ch=0x" + Integer.toHexString(s2.charAt(idx)));
+      }
+      */
+    }
+
+    final FixedBitSet isS1 = new FixedBitSet(NUM_DOCS);
+    for(int idx=0;idx<NUM_DOCS;idx++) {
+      if (random.nextBoolean()) {
+        isS1.set(idx);
+      }
+    }
+
+    final IndexReader r;
+    if (true) { 
+      final IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))
+        .setOpenMode(IndexWriterConfig.OpenMode.CREATE)
+        .setMergePolicy(newLogMergePolicy());
+      iwc.setRAMBufferSizeMB(16.0 + 16.0 * random.nextDouble());
+      iwc.setMaxBufferedDocs(-1);
+      final RandomIndexWriter riw = new RandomIndexWriter(random, dir, iwc);
+
+      for(int idx=0;idx<NUM_DOCS;idx++) {
+        final Document doc = new Document();
+        String s = isS1.get(idx) ? s1 : s2;
+        final Field f = newField("field", s, Field.Index.ANALYZED);
+        f.setIndexOptions(options);
+        final int count = _TestUtil.nextInt(random, 1, 4);
+        for(int ct=0;ct<count;ct++) {
+          doc.add(f);
+        }
+        riw.addDocument(doc);
+      }
+
+      r = riw.getReader();
+      riw.close();
+    } else {
+      r = IndexReader.open(dir);
+    }
+
+    /*
+    if (VERBOSE) {
+      System.out.println("TEST: terms");
+      TermEnum termEnum = r.terms();
+      while(termEnum.next()) {
+        System.out.println("  term=" + termEnum.term() + " len=" + termEnum.term().text().length());
+        assertTrue(termEnum.docFreq() > 0);
+        System.out.println("    s1?=" + (termEnum.term().text().equals(s1)) + " s1len=" + s1.length());
+        System.out.println("    s2?=" + (termEnum.term().text().equals(s2)) + " s2len=" + s2.length());
+        final String s = termEnum.term().text();
+        for(int idx=0;idx<s.length();idx++) {
+          System.out.println("      ch=0x" + Integer.toHexString(s.charAt(idx)));
+        }
+      }
+    }
+    */
+
+    assertEquals(NUM_DOCS, r.numDocs());
+    assertTrue(r.docFreq(new Term("field", s1)) > 0);
+    assertTrue(r.docFreq(new Term("field", s2)) > 0);
+
+    int num = atLeast(1000);
+    for(int iter=0;iter<num;iter++) {
+
+      final String term;
+      final boolean doS1;
+      if (random.nextBoolean()) {
+        term = s1;
+        doS1 = true;
+      } else {
+        term = s2;
+        doS1 = false;
+      }
+
+      if (VERBOSE) {
+        System.out.println("\nTEST: iter=" + iter + " doS1=" + doS1);
+      }
+        
+      final DocsEnum postings = MultiFields.getTermDocsEnum(r, null, "field", new BytesRef(term));
+
+      int docID = -1;
+      while(docID < DocsEnum.NO_MORE_DOCS) {
+        final int what = random.nextInt(3);
+        if (what == 0) {
+          if (VERBOSE) {
+            System.out.println("TEST: docID=" + docID + "; do next()");
+          }
+          // nextDoc
+          int expected = docID+1;
+          while(true) {
+            if (expected == NUM_DOCS) {
+              expected = Integer.MAX_VALUE;
+              break;
+            } else if (isS1.get(expected) == doS1) {
+              break;
+            } else {
+              expected++;
+            }
+          }
+          docID = postings.nextDoc();
+          if (VERBOSE) {
+            System.out.println("  got docID=" + docID);
+          }
+          assertEquals(expected, docID);
+          if (docID == DocsEnum.NO_MORE_DOCS) {
+            break;
+          }
+
+          if (random.nextInt(6) == 3) {
+            final int freq = postings.freq();
+            assertTrue(freq >=1 && freq <= 4);
+          }
+        } else {
+          // advance
+          final int targetDocID;
+          if (docID == -1) {
+            targetDocID = random.nextInt(NUM_DOCS+1);
+          } else {
+            targetDocID = docID + _TestUtil.nextInt(random, 1, NUM_DOCS - docID);
+          }
+          if (VERBOSE) {
+            System.out.println("TEST: docID=" + docID + "; do advance(" + targetDocID + ")");
+          }
+          int expected = targetDocID;
+          while(true) {
+            if (expected == NUM_DOCS) {
+              expected = Integer.MAX_VALUE;
+              break;
+            } else if (isS1.get(expected) == doS1) {
+              break;
+            } else {
+              expected++;
+            }
+          }
+          
+          docID = postings.advance(targetDocID);
+          if (VERBOSE) {
+            System.out.println("  got docID=" + docID);
+          }
+          assertEquals(expected, docID);
+          if (docID == DocsEnum.NO_MORE_DOCS) {
+            break;
+          }
+          
+          if (random.nextInt(6) == 3) {
+            final int freq = postings.freq();
+            assertTrue(freq >=1 && freq <= 4);
+          }
+        }
+      }
+    }
+    r.close();
+    dir.close();
+  }
 }

Modified: lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/TestOmitTf.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/TestOmitTf.java?rev=1147873&r1=1147872&r2=1147873&view=diff
==============================================================================
--- lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/TestOmitTf.java (original)
+++ lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/TestOmitTf.java Mon Jul 18 13:34:29 2011
@@ -19,6 +19,7 @@ package org.apache.lucene.index;
 
 import java.io.IOException;
 
+import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.util.TermContext;
 import org.apache.lucene.util._TestUtil;
@@ -26,6 +27,7 @@ import org.apache.lucene.analysis.Analyz
 import org.apache.lucene.analysis.MockAnalyzer;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
+import org.apache.lucene.index.FieldInfo.IndexOptions;
 import org.apache.lucene.index.IndexReader.AtomicReaderContext;
 import org.apache.lucene.search.*;
 import org.apache.lucene.search.BooleanClause.Occur;
@@ -47,6 +49,7 @@ public class TestOmitTf extends LuceneTe
         @Override public Explanation idfExplain(TermContext[] terms, IndexSearcher searcher) throws IOException {
           return new Explanation(1.0f, "Inexplicable");
         }
+        @Override public float scorePayload(int doc, int start, int end, BytesRef payload) { return 1.0f; }
       };
     }
   }
@@ -65,7 +68,7 @@ public class TestOmitTf extends LuceneTe
        
     // this field will NOT have Tf
     Field f2 = newField("f2", "This field has NO Tf in all docs", Field.Store.NO, Field.Index.ANALYZED);
-    f2.setOmitTermFreqAndPositions(true);
+    f2.setIndexOptions(IndexOptions.DOCS_ONLY);
     d.add(f2);
         
     writer.addDocument(d);
@@ -75,10 +78,10 @@ public class TestOmitTf extends LuceneTe
     d = new Document();
         
     // Reverse
-    f1.setOmitTermFreqAndPositions(true);
+    f1.setIndexOptions(IndexOptions.DOCS_ONLY);
     d.add(f1);
         
-    f2.setOmitTermFreqAndPositions(false);        
+    f2.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);        
     d.add(f2);
         
     writer.addDocument(d);
@@ -90,8 +93,8 @@ public class TestOmitTf extends LuceneTe
 
     SegmentReader reader = getOnlySegmentReader(IndexReader.open(ram, false));
     FieldInfos fi = reader.fieldInfos();
-    assertTrue("OmitTermFreqAndPositions field bit should be set.", fi.fieldInfo("f1").omitTermFreqAndPositions);
-    assertTrue("OmitTermFreqAndPositions field bit should be set.", fi.fieldInfo("f2").omitTermFreqAndPositions);
+    assertEquals("OmitTermFreqAndPositions field bit should be set.", IndexOptions.DOCS_ONLY, fi.fieldInfo("f1").indexOptions);
+    assertEquals("OmitTermFreqAndPositions field bit should be set.", IndexOptions.DOCS_ONLY, fi.fieldInfo("f2").indexOptions);
         
     reader.close();
     ram.close();
@@ -117,7 +120,7 @@ public class TestOmitTf extends LuceneTe
        
     // this field will NOT have Tf
     Field f2 = newField("f2", "This field has NO Tf in all docs", Field.Store.NO, Field.Index.ANALYZED);
-    f2.setOmitTermFreqAndPositions(true);
+    f2.setIndexOptions(IndexOptions.DOCS_ONLY);
     d.add(f2);
 
     for(int i=0;i<30;i++)
@@ -128,10 +131,10 @@ public class TestOmitTf extends LuceneTe
     d = new Document();
         
     // Reverese
-    f1.setOmitTermFreqAndPositions(true);
+    f1.setIndexOptions(IndexOptions.DOCS_ONLY);
     d.add(f1);
         
-    f2.setOmitTermFreqAndPositions(false);        
+    f2.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);        
     d.add(f2);
         
     for(int i=0;i<30;i++)
@@ -144,8 +147,8 @@ public class TestOmitTf extends LuceneTe
 
     SegmentReader reader = getOnlySegmentReader(IndexReader.open(ram, false));
     FieldInfos fi = reader.fieldInfos();
-    assertTrue("OmitTermFreqAndPositions field bit should be set.", fi.fieldInfo("f1").omitTermFreqAndPositions);
-    assertTrue("OmitTermFreqAndPositions field bit should be set.", fi.fieldInfo("f2").omitTermFreqAndPositions);
+    assertEquals("OmitTermFreqAndPositions field bit should be set.", IndexOptions.DOCS_ONLY, fi.fieldInfo("f1").indexOptions);
+    assertEquals("OmitTermFreqAndPositions field bit should be set.", IndexOptions.DOCS_ONLY, fi.fieldInfo("f2").indexOptions);
         
     reader.close();
     ram.close();
@@ -176,7 +179,7 @@ public class TestOmitTf extends LuceneTe
     for(int i=0;i<5;i++)
       writer.addDocument(d);
 
-    f2.setOmitTermFreqAndPositions(true);
+    f2.setIndexOptions(IndexOptions.DOCS_ONLY);
         
     for(int i=0;i<20;i++)
       writer.addDocument(d);
@@ -189,8 +192,8 @@ public class TestOmitTf extends LuceneTe
 
     SegmentReader reader = getOnlySegmentReader(IndexReader.open(ram, false));
     FieldInfos fi = reader.fieldInfos();
-    assertTrue("OmitTermFreqAndPositions field bit should not be set.", !fi.fieldInfo("f1").omitTermFreqAndPositions);
-    assertTrue("OmitTermFreqAndPositions field bit should be set.", fi.fieldInfo("f2").omitTermFreqAndPositions);
+    assertEquals("OmitTermFreqAndPositions field bit should not be set.", IndexOptions.DOCS_AND_FREQS_AND_POSITIONS, fi.fieldInfo("f1").indexOptions);
+    assertEquals("OmitTermFreqAndPositions field bit should be set.", IndexOptions.DOCS_ONLY, fi.fieldInfo("f2").indexOptions);
         
     reader.close();
     ram.close();
@@ -198,8 +201,10 @@ public class TestOmitTf extends LuceneTe
 
   private void assertNoPrx(Directory dir) throws Throwable {
     final String[] files = dir.listAll();
-    for(int i=0;i<files.length;i++)
+    for(int i=0;i<files.length;i++) {
       assertFalse(files[i].endsWith(".prx"));
+      assertFalse(files[i].endsWith(".pos"));
+    }
   }
 
   // Verifies no *.prx exists when all fields omit term freq:
@@ -213,8 +218,8 @@ public class TestOmitTf extends LuceneTe
     lmp.setUseCompoundFile(false);
     Document d = new Document();
         
-    Field f1 = newField("f1", "This field has term freqs", Field.Store.NO, Field.Index.ANALYZED);
-    f1.setOmitTermFreqAndPositions(true);
+    Field f1 = newField("f1", "This field has no term freqs", Field.Store.NO, Field.Index.ANALYZED);
+    f1.setIndexOptions(IndexOptions.DOCS_ONLY);
     d.add(f1);
 
     for(int i=0;i<30;i++)
@@ -223,7 +228,15 @@ public class TestOmitTf extends LuceneTe
     writer.commit();
 
     assertNoPrx(ram);
-        
+    
+    // now add some documents with positions, and check there is no prox after optimization
+    d = new Document();
+    f1 = newField("f1", "This field has positions", Field.Store.NO, Field.Index.ANALYZED);
+    d.add(f1);
+    
+    for(int i=0;i<30;i++)
+      writer.addDocument(d);
+ 
     // force merge
     writer.optimize();
     // flush
@@ -253,7 +266,7 @@ public class TestOmitTf extends LuceneTe
       sb.append(term).append(" ");
       String content  = sb.toString();
       Field noTf = newField("noTf", content + (i%2==0 ? "" : " notf"), Field.Store.NO, Field.Index.ANALYZED);
-      noTf.setOmitTermFreqAndPositions(true);
+      noTf.setIndexOptions(IndexOptions.DOCS_ONLY);
       d.add(noTf);
           
       Field tf = newField("tf", content + (i%2==0 ? " tf" : ""), Field.Store.NO, Field.Index.ANALYZED);

Modified: lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/TestSameTokenSamePosition.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/TestSameTokenSamePosition.java?rev=1147873&r1=1147872&r2=1147873&view=diff
==============================================================================
--- lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/TestSameTokenSamePosition.java (original)
+++ lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/TestSameTokenSamePosition.java Mon Jul 18 13:34:29 2011
@@ -47,6 +47,22 @@ public class TestSameTokenSamePosition e
     riw.close();
     dir.close();
   }
+  
+  /**
+   * Same as the above, but with more docs
+   */
+  public void testMoreDocs() throws Exception {
+    Directory dir = newDirectory();
+    RandomIndexWriter riw = new RandomIndexWriter(random, dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new BugReproAnalyzer()));
+    Document doc = new Document();
+    doc.add(new Field("eng", "Six drunken" /*This shouldn't matter. */, 
+                      Field.Store.YES, Field.Index.ANALYZED));
+    for (int i = 0; i < 100; i++) {
+      riw.addDocument(doc);
+    }
+    riw.close();
+    dir.close();
+  }
 }
 
 final class BugReproAnalyzer extends Analyzer{

Modified: lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/values/TestDocValuesIndexing.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/values/TestDocValuesIndexing.java?rev=1147873&r1=1147872&r2=1147873&view=diff
==============================================================================
--- lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/values/TestDocValuesIndexing.java (original)
+++ lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/values/TestDocValuesIndexing.java Mon Jul 18 13:34:29 2011
@@ -49,7 +49,7 @@ import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.FloatsRef;
 import org.apache.lucene.util.LongsRef;
 import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.util.OpenBitSet;
+import org.apache.lucene.util.FixedBitSet;
 import org.apache.lucene.util._TestUtil;
 import org.junit.Before;
 
@@ -233,11 +233,11 @@ public class TestDocValuesIndexing exten
     // run in random order to test if fill works correctly during merges
     Collections.shuffle(numVariantList, random);
     for (ValueType val : numVariantList) {
-      OpenBitSet deleted = indexValues(w, numValues, val, numVariantList,
+      FixedBitSet deleted = indexValues(w, numValues, val, numVariantList,
           withDeletions, 7);
       List<Closeable> closeables = new ArrayList<Closeable>();
       IndexReader r = IndexReader.open(w, true);
-      final int numRemainingValues = (int) (numValues - deleted.cardinality());
+      final int numRemainingValues = numValues - deleted.cardinality();
       final int base = r.numDocs() - numRemainingValues;
       // for FIXED_INTS_8 we use value mod 128 - to enable testing in 
       // one go we simply use numValues as the mod for all other INT types
@@ -331,11 +331,11 @@ public class TestDocValuesIndexing exten
     for (ValueType byteIndexValue : byteVariantList) {
       List<Closeable> closeables = new ArrayList<Closeable>();
       final int bytesSize = 1 + atLeast(50);
-      OpenBitSet deleted = indexValues(w, numValues, byteIndexValue,
+      FixedBitSet deleted = indexValues(w, numValues, byteIndexValue,
           byteVariantList, withDeletions, bytesSize);
       final IndexReader r = IndexReader.open(w, withDeletions);
       assertEquals(0, r.numDeletedDocs());
-      final int numRemainingValues = (int) (numValues - deleted.cardinality());
+      final int numRemainingValues = numValues - deleted.cardinality();
       final int base = r.numDocs() - numRemainingValues;
       IndexDocValues bytesReader = getDocValues(r, byteIndexValue.name());
       assertNotNull("field " + byteIndexValue.name()
@@ -484,11 +484,11 @@ public class TestDocValuesIndexing exten
       Index.ANALYZED_NO_NORMS, Index.NOT_ANALYZED, Index.NOT_ANALYZED_NO_NORMS,
       Index.NO };
 
-  private OpenBitSet indexValues(IndexWriter w, int numValues, ValueType value,
+  private FixedBitSet indexValues(IndexWriter w, int numValues, ValueType value,
       List<ValueType> valueVarList, boolean withDeletions, int bytesSize)
       throws CorruptIndexException, IOException {
     final boolean isNumeric = NUMERICS.contains(value);
-    OpenBitSet deleted = new OpenBitSet(numValues);
+    FixedBitSet deleted = new FixedBitSet(numValues);
     Document doc = new Document();
     Index idx = IDX_VALUES[random.nextInt(IDX_VALUES.length)];
     AbstractField field = random.nextBoolean() ? new IndexDocValuesField(value.name())

Modified: lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/search/BaseTestRangeFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/search/BaseTestRangeFilter.java?rev=1147873&r1=1147872&r2=1147873&view=diff
==============================================================================
--- lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/search/BaseTestRangeFilter.java (original)
+++ lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/search/BaseTestRangeFilter.java Mon Jul 18 13:34:29 2011
@@ -67,7 +67,7 @@ public class BaseTestRangeFilter extends
   static TestIndex unsignedIndexDir;
   
   static int minId = 0;
-  static int maxId = atLeast(500);
+  static int maxId;
   
   static final int intLength = Integer.toString(Integer.MAX_VALUE).length();
   
@@ -93,6 +93,7 @@ public class BaseTestRangeFilter extends
   
   @BeforeClass
   public static void beforeClassBaseTestRangeFilter() throws Exception {
+    maxId = atLeast(500);
     signedIndexDir = new TestIndex(random, Integer.MAX_VALUE, Integer.MIN_VALUE, true);
     unsignedIndexDir = new TestIndex(random, Integer.MAX_VALUE, 0, false);
     signedIndexReader = build(random, signedIndexDir);

Modified: lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/search/JustCompileSearch.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/search/JustCompileSearch.java?rev=1147873&r1=1147872&r2=1147873&view=diff
==============================================================================
--- lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/search/JustCompileSearch.java (original)
+++ lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/search/JustCompileSearch.java Mon Jul 18 13:34:29 2011
@@ -265,11 +265,6 @@ final class JustCompileSearch {
     public byte computeNorm(FieldInvertState state) {
       throw new UnsupportedOperationException(UNSUPPORTED_MSG);
     }
-
-    @Override
-    public float sloppyFreq(int distance) {
-      throw new UnsupportedOperationException(UNSUPPORTED_MSG);
-    }
   }
   
   static final class JustCompileSimilarityProvider implements SimilarityProvider {

Modified: lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/search/TestCachingWrapperFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/search/TestCachingWrapperFilter.java?rev=1147873&r1=1147872&r2=1147873&view=diff
==============================================================================
--- lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/search/TestCachingWrapperFilter.java (original)
+++ lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/search/TestCachingWrapperFilter.java Mon Jul 18 13:34:29 2011
@@ -30,8 +30,7 @@ import org.apache.lucene.index.SlowMulti
 import org.apache.lucene.index.Term;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.util.OpenBitSet;
-import org.apache.lucene.util.OpenBitSetDISI;
+import org.apache.lucene.util.FixedBitSet;
 import org.apache.lucene.util._TestUtil;
 
 public class TestCachingWrapperFilter extends LuceneTestCase {
@@ -125,7 +124,7 @@ public class TestCachingWrapperFilter ex
     if (originalSet.isCacheable()) {
       assertEquals("Cached DocIdSet must be of same class like uncached, if cacheable", originalSet.getClass(), cachedSet.getClass());
     } else {
-      assertTrue("Cached DocIdSet must be an OpenBitSet if the original one was not cacheable", cachedSet instanceof OpenBitSetDISI || cachedSet == DocIdSet.EMPTY_DOCIDSET);
+      assertTrue("Cached DocIdSet must be an FixedBitSet if the original one was not cacheable", cachedSet instanceof FixedBitSet || cachedSet == DocIdSet.EMPTY_DOCIDSET);
     }
   }
   
@@ -143,11 +142,11 @@ public class TestCachingWrapperFilter ex
     assertDocIdSetCacheable(reader, NumericRangeFilter.newIntRange("test", Integer.valueOf(10000), Integer.valueOf(-10000), true, true), true);
     // is cacheable:
     assertDocIdSetCacheable(reader, FieldCacheRangeFilter.newIntRange("test", Integer.valueOf(10), Integer.valueOf(20), true, true), true);
-    // a openbitset filter is always cacheable
+    // a fixedbitset filter is always cacheable
     assertDocIdSetCacheable(reader, new Filter() {
       @Override
       public DocIdSet getDocIdSet(AtomicReaderContext context) {
-        return new OpenBitSet();
+        return new FixedBitSet(context.reader.maxDoc());
       }
     }, true);
 

Modified: lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/search/TestCustomSearcherSort.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/search/TestCustomSearcherSort.java?rev=1147873&r1=1147872&r2=1147873&view=diff
==============================================================================
--- lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/search/TestCustomSearcherSort.java (original)
+++ lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/search/TestCustomSearcherSort.java Mon Jul 18 13:34:29 2011
@@ -39,7 +39,7 @@ public class TestCustomSearcherSort exte
   private IndexReader reader;
   private Query query = null;
   // reduced from 20000 to 2000 to speed up test...
-  private final static int INDEX_SIZE = atLeast(2000);
+  private int INDEX_SIZE;
   
   /**
    * Create index and query for test cases.
@@ -47,6 +47,7 @@ public class TestCustomSearcherSort exte
   @Override
   public void setUp() throws Exception {
     super.setUp();
+    INDEX_SIZE = atLeast(2000);
     index = newDirectory();
     RandomIndexWriter writer = new RandomIndexWriter(random, index);
     RandomGen random = new RandomGen(this.random);

Modified: lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/search/TestDocValuesScoring.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/search/TestDocValuesScoring.java?rev=1147873&r1=1147872&r2=1147873&view=diff
==============================================================================
--- lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/search/TestDocValuesScoring.java (original)
+++ lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/search/TestDocValuesScoring.java Mon Jul 18 13:34:29 2011
@@ -30,6 +30,7 @@ import org.apache.lucene.index.IndexRead
 import org.apache.lucene.index.codecs.CodecProvider;
 import org.apache.lucene.index.values.IndexDocValues.Source;
 import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.util.TermContext;
 
@@ -145,11 +146,6 @@ public class TestDocValuesScoring extend
     }
 
     @Override
-    public float sloppyFreq(int distance) {
-      return sim.sloppyFreq(distance);
-    }
-
-    @Override
     public Stats computeStats(IndexSearcher searcher, String fieldName, float queryBoost, TermContext... termContexts) throws IOException {
       return sim.computeStats(searcher, fieldName, queryBoost, termContexts);
     }
@@ -189,6 +185,16 @@ public class TestDocValuesScoring extend
         }
         
         @Override
+        public float computeSlopFactor(int distance) {
+          return sub.computeSlopFactor(distance);
+        }
+
+        @Override
+        public float computePayloadFactor(int doc, int start, int end, BytesRef payload) {
+          return sub.computePayloadFactor(doc, start, end, payload);
+        }
+
+        @Override
         public Explanation explain(int doc, Explanation freq) {
           Explanation boostExplanation = new Explanation((float) values.getFloat(doc), "indexDocValue(" + boostField + ")");
           Explanation simExplanation = sub.explain(doc, freq);



Mime
View raw message