lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From rm...@apache.org
Subject svn commit: r1143840 [5/22] - in /lucene/dev/branches/flexscoring: ./ dev-tools/eclipse/ dev-tools/idea/.idea/ dev-tools/idea/lucene/contrib/ dev-tools/idea/lucene/contrib/demo/ dev-tools/idea/lucene/contrib/highlighter/ dev-tools/idea/lucene/contrib/i...
Date Thu, 07 Jul 2011 14:18:33 GMT
Modified: lucene/dev/branches/flexscoring/lucene/contrib/spatial/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/contrib/spatial/build.xml?rev=1143840&r1=1143839&r2=1143840&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/contrib/spatial/build.xml (original)
+++ lucene/dev/branches/flexscoring/lucene/contrib/spatial/build.xml Thu Jul  7 14:18:02 2011
@@ -25,7 +25,7 @@
 
   <import file="../contrib-build.xml"/>
 
-  <contrib-uptodate name="queries" property="queries.uptodate" classpath.property="queries.jar"/>
+  <contrib-uptodate name="queries-contrib" contrib-src-name="queries" property="queries.uptodate" classpath.property="queries.jar" />
 
   <path id="classpath">
     <pathelement path="${queries.jar}"/>

Modified: lucene/dev/branches/flexscoring/lucene/contrib/spatial/src/java/org/apache/lucene/spatial/tier/CartesianShapeFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/contrib/spatial/src/java/org/apache/lucene/spatial/tier/CartesianShapeFilter.java?rev=1143840&r1=1143839&r2=1143840&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/contrib/spatial/src/java/org/apache/lucene/spatial/tier/CartesianShapeFilter.java (original)
+++ lucene/dev/branches/flexscoring/lucene/contrib/spatial/src/java/org/apache/lucene/spatial/tier/CartesianShapeFilter.java Thu Jul  7 14:18:02 2011
@@ -46,7 +46,7 @@ public class CartesianShapeFilter extend
   
   @Override
   public DocIdSet getDocIdSet(final AtomicReaderContext context) throws IOException {
-    final Bits delDocs = context.reader.getDeletedDocs();
+    final Bits liveDocs = context.reader.getLiveDocs();
     final List<Double> area = shape.getArea();
     final int sz = area.size();
     
@@ -58,7 +58,7 @@ public class CartesianShapeFilter extend
       return new DocIdSet() {
         @Override
         public DocIdSetIterator iterator() throws IOException {
-          return context.reader.termDocsEnum(delDocs, fieldName, bytesRef);
+          return context.reader.termDocsEnum(liveDocs, fieldName, bytesRef);
         }
         
         @Override
@@ -71,7 +71,7 @@ public class CartesianShapeFilter extend
       for (int i =0; i< sz; i++) {
         double boxId = area.get(i).doubleValue();
         NumericUtils.longToPrefixCoded(NumericUtils.doubleToSortableLong(boxId), 0, bytesRef);
-        final DocsEnum docsEnum = context.reader.termDocsEnum(delDocs, fieldName, bytesRef);
+        final DocsEnum docsEnum = context.reader.termDocsEnum(liveDocs, fieldName, bytesRef);
         if (docsEnum == null) continue;
         // iterate through all documents
         // which have this boxId

Modified: lucene/dev/branches/flexscoring/lucene/contrib/spatial/src/test/org/apache/lucene/spatial/tier/TestCartesian.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/contrib/spatial/src/test/org/apache/lucene/spatial/tier/TestCartesian.java?rev=1143840&r1=1143839&r2=1143840&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/contrib/spatial/src/test/org/apache/lucene/spatial/tier/TestCartesian.java (original)
+++ lucene/dev/branches/flexscoring/lucene/contrib/spatial/src/test/org/apache/lucene/spatial/tier/TestCartesian.java Thu Jul  7 14:18:02 2011
@@ -26,7 +26,7 @@ import org.apache.lucene.document.Docume
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.NumericField;
 import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexReader.AtomicReaderContext;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.Query;
@@ -35,10 +35,6 @@ import org.apache.lucene.search.Sort;
 import org.apache.lucene.search.SortField;
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.search.TopDocs;
-import org.apache.lucene.search.function.CustomScoreQuery;
-import org.apache.lucene.search.function.CustomScoreProvider;
-import org.apache.lucene.search.function.FieldScoreQuery;
-import org.apache.lucene.search.function.FieldScoreQuery.Type;
 import org.apache.lucene.spatial.DistanceUtils;
 import org.apache.lucene.spatial.geohash.GeoHashUtils;
 import org.apache.lucene.spatial.geometry.DistanceUnits;
@@ -216,33 +212,6 @@ public class TestCartesian extends Lucen
     if (VERBOSE) System.out.println(dq);
     //create a term query to search against all documents
     Query tq = new TermQuery(new Term("metafile", "doc"));
-
-    FieldScoreQuery fsQuery = new FieldScoreQuery("geo_distance", Type.FLOAT);
-
-    CustomScoreQuery customScore = new CustomScoreQuery(dq.getQuery(tq),fsQuery){
-
-      @Override
-      protected CustomScoreProvider getCustomScoreProvider(IndexReader reader) {
-        return new CustomScoreProvider(reader) {
-          @Override // TODO: broken, as reader is not used!
-          public float customScore(int doc, float subQueryScore, float valSrcScore){
-            if (VERBOSE) System.out.println(doc);
-            if (dq.distanceFilter.getDistance(doc) == null)
-              return 0;
-
-            double distance = dq.distanceFilter.getDistance(doc);
-            // boost score shouldn't exceed 1
-            if (distance < 1.0d)
-              distance = 1.0d;
-            //boost by distance is invertly proportional to
-            // to distance from center point to location
-            float score = (float) ((miles - distance) / miles );
-            return score * subQueryScore;
-          }
-        };
-      }
-      
-    };
     // Create a distance sort
     // As the radius filter has performed the distance calculations
     // already, pass in the filter to reuse the results.
@@ -252,7 +221,7 @@ public class TestCartesian extends Lucen
 
     // Perform the search, using the term query, the serial chain filter, and the
     // distance sort
-    TopDocs hits = searcher.search(customScore,null, 1000, sort);
+    TopDocs hits = searcher.search(dq.getQuery(tq),null, 1000, sort);
     int results = hits.totalHits;
     ScoreDoc[] scoreDocs = hits.scoreDocs; 
     
@@ -312,33 +281,6 @@ public class TestCartesian extends Lucen
     if (VERBOSE) System.out.println(dq);
     //create a term query to search against all documents
     Query tq = new TermQuery(new Term("metafile", "doc"));
-
-    FieldScoreQuery fsQuery = new FieldScoreQuery("geo_distance", Type.FLOAT);
-
-    CustomScoreQuery customScore = new CustomScoreQuery(dq.getQuery(tq),fsQuery){
-
-      @Override
-      protected CustomScoreProvider getCustomScoreProvider(IndexReader reader) {
-        return new CustomScoreProvider(reader) {
-          @Override // TODO: broken, as reader is not used!
-          public float customScore(int doc, float subQueryScore, float valSrcScore){
-            if (VERBOSE) System.out.println(doc);
-            if (dq.distanceFilter.getDistance(doc) == null)
-              return 0;
-
-            double distance = dq.distanceFilter.getDistance(doc);
-            // boost score shouldn't exceed 1
-            if (distance < 1.0d)
-              distance = 1.0d;
-            //boost by distance is invertly proportional to
-            // to distance from center point to location
-            float score = (float) ((miles - distance) / miles );
-            return score * subQueryScore;
-          }
-        };
-      }
-      
-    };
     // Create a distance sort
     // As the radius filter has performed the distance calculations
     // already, pass in the filter to reuse the results.
@@ -348,7 +290,7 @@ public class TestCartesian extends Lucen
 
     // Perform the search, using the term query, the serial chain filter, and the
     // distance sort
-    TopDocs hits = searcher.search(customScore,null, 1000, sort);
+    TopDocs hits = searcher.search(dq.getQuery(tq),null, 1000, sort);
     int results = hits.totalHits;
     ScoreDoc[] scoreDocs = hits.scoreDocs; 
 
@@ -410,31 +352,6 @@ public class TestCartesian extends Lucen
       if (VERBOSE) System.out.println(dq);
       //create a term query to search against all documents
       Query tq = new TermQuery(new Term("metafile", "doc"));
-    
-      FieldScoreQuery fsQuery = new FieldScoreQuery("geo_distance", Type.FLOAT);
-    
-      CustomScoreQuery customScore = new CustomScoreQuery(dq.getQuery(tq),fsQuery){
-        @Override
-        protected CustomScoreProvider getCustomScoreProvider(IndexReader reader) {
-          return new CustomScoreProvider(reader) {
-            @Override // TODO: broken, as reader is not used!
-            public float customScore(int doc, float subQueryScore, float valSrcScore){
-              if (VERBOSE) System.out.println(doc);
-              if (dq.distanceFilter.getDistance(doc) == null)
-                return 0;
-          
-              double distance = dq.distanceFilter.getDistance(doc);
-              // boost score shouldn't exceed 1
-              if (distance < 1.0d)
-                distance = 1.0d;
-              //boost by distance is invertly proportional to
-              // to distance from center point to location
-              float score = (float) ( (miles - distance) / miles );
-              return score * subQueryScore;
-            }
-          };
-        }
-      };
       // Create a distance sort
       // As the radius filter has performed the distance calculations
       // already, pass in the filter to reuse the results.
@@ -444,7 +361,7 @@ public class TestCartesian extends Lucen
     
       // Perform the search, using the term query, the serial chain filter, and the
       // distance sort
-      TopDocs hits = searcher.search(customScore,null, 1000, sort);
+      TopDocs hits = searcher.search(dq.getQuery(tq),null, 1000, sort);
       int results = hits.totalHits;
       ScoreDoc[] scoreDocs = hits.scoreDocs; 
     
@@ -506,30 +423,6 @@ public class TestCartesian extends Lucen
       if (VERBOSE) System.out.println(dq);
       //create a term query to search against all documents
       Query tq = new TermQuery(new Term("metafile", "doc"));
-	    
-      FieldScoreQuery fsQuery = new FieldScoreQuery("geo_distance", Type.FLOAT);
-      CustomScoreQuery customScore = new CustomScoreQuery(tq,fsQuery){
-        @Override
-        protected CustomScoreProvider getCustomScoreProvider(IndexReader reader) {
-          return new CustomScoreProvider(reader) {
-              @Override // TODO: broken, as reader is not used!
-              public float customScore(int doc, float subQueryScore, float valSrcScore){
-              if (VERBOSE) System.out.println(doc);
-              if (dq.distanceFilter.getDistance(doc) == null)
-                return 0;
-            
-              double distance = dq.distanceFilter.getDistance(doc);
-              // boost score shouldn't exceed 1
-              if (distance < 1.0d)
-                distance = 1.0d;
-              //boost by distance is invertly proportional to
-              // to distance from center point to location
-              float score = (float) ( (miles - distance) / miles );
-              return score * subQueryScore;
-            }
-          };
-        }
-      };
       // Create a distance sort
       // As the radius filter has performed the distance calculations
       // already, pass in the filter to reuse the results.
@@ -539,7 +432,7 @@ public class TestCartesian extends Lucen
 	    
       // Perform the search, using the term query, the serial chain filter, and the
       // distance sort
-      TopDocs hits = searcher.search(customScore,dq.getFilter(), 1000); //,sort);
+      TopDocs hits = searcher.search(tq,dq.getFilter(), 1000); //,sort);
       int results = hits.totalHits;
       ScoreDoc[] scoreDocs = hits.scoreDocs; 
 	    

Modified: lucene/dev/branches/flexscoring/lucene/contrib/wordnet/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/contrib/wordnet/build.xml?rev=1143840&r1=1143839&r2=1143840&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/contrib/wordnet/build.xml (original)
+++ lucene/dev/branches/flexscoring/lucene/contrib/wordnet/build.xml Thu Jul  7 14:18:02 2011
@@ -30,22 +30,6 @@
 
   <import file="../contrib-build.xml"/>
 
-  <module-uptodate name="analysis/common" jarfile="${common.dir}/../modules/analysis/build/common/lucene-analyzers-common-${version}.jar"
-      property="analyzers-common.uptodate" classpath.property="analyzers-common.jar"/>
-
-  <path id="classpath">
-	 <pathelement path="${analyzers-common.jar}"/>
-	 <path refid="base.classpath"/>
-  </path>
-
-  <target name="compile-core" depends="compile-analyzers-common, common.compile-core" />
-
-  <target name="compile-analyzers-common" unless="analyzers-common.uptodate">
-    <subant target="default">
-      <fileset dir="${common.dir}/../modules/analysis/common" includes="build.xml"/>
-    </subant>
-  </target>
-	
   <target name="index" depends="compile" description="Build WordNet index">
     <fail if="synindex.exists">
       Index already exists - must remove first.
@@ -83,24 +67,4 @@
     </java>
   </target>
 
-  <target name="expand" description="Perform synonym expansion on a query">
-    <fail unless="synindex.exists">
-      Index does not exist.
-    </fail>
-
-    <fail unless="query">
-      Must specify 'query' property.
-    </fail>
-    
-    <java classname="org.apache.lucene.wordnet.SynExpand">
-      <classpath>
-        <path refid="compile.classpath"/>
-        <pathelement location="${build.dir}/classes"/>
-      </classpath>
-
-      <arg file="${synindex.dir}"/>
-      <arg value="${query}"/>
-    </java>
-  </target>
-
 </project>

Modified: lucene/dev/branches/flexscoring/lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/SynExpand.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/SynExpand.java?rev=1143840&r1=1143839&r2=1143840&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/SynExpand.java (original)
+++ lucene/dev/branches/flexscoring/lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/SynExpand.java Thu Jul  7 14:18:02 2011
@@ -17,7 +17,6 @@ package org.apache.lucene.wordnet;
  * limitations under the License.
  */
 
-import java.io.File;
 import java.io.IOException;
 import java.io.StringReader;
 import java.util.HashSet;
@@ -28,7 +27,6 @@ import java.util.Set;
 
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.standard.StandardAnalyzer;
 import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.index.IndexReader;
@@ -41,8 +39,6 @@ import org.apache.lucene.search.IndexSea
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.Scorer;
 import org.apache.lucene.search.TermQuery;
-import org.apache.lucene.store.FSDirectory;
-import org.apache.lucene.util.Version;
 
 
 /**
@@ -54,48 +50,13 @@ import org.apache.lucene.util.Version;
 public final class SynExpand {
 
 	/**
-	 * Test driver for synonym expansion.
-	 * Uses boost factor of 0.9 for illustrative purposes.
-	 *
-	 * If you pass in the query "big dog" then it prints out:
-	 *
-	 * <code><pre>
-	 * Query: big adult^0.9 bad^0.9 bighearted^0.9 boastful^0.9 boastfully^0.9 bounteous^0.9 bountiful^0.9 braggy^0.9 crowing^0.9 freehanded^0.9 giving^0.9 grown^0.9 grownup^0.9 handsome^0.9 large^0.9 liberal^0.9 magnanimous^0.9 momentous^0.9 openhanded^0.9 prominent^0.9 swelled^0.9 vainglorious^0.9 vauntingly^0.9
-	 * dog andiron^0.9 blackguard^0.9 bounder^0.9 cad^0.9 chase^0.9 click^0.9 detent^0.9 dogtooth^0.9 firedog^0.9 frank^0.9 frankfurter^0.9 frump^0.9 heel^0.9 hotdog^0.9 hound^0.9 pawl^0.9 tag^0.9 tail^0.9 track^0.9 trail^0.9 weenie^0.9 wiener^0.9 wienerwurst^0.9
-	 * </pre></code>
-	 */
-	public static void main(String[] args) throws IOException
-	{
-		if (args.length != 2)
-		{
-			System.out.println(
-							   "java org.apache.lucene.wordnet.SynExpand <index path> <query>");
-		}
-
-		FSDirectory directory = FSDirectory.open(new File(args[0]));
-		IndexSearcher searcher = new IndexSearcher(directory, true);
-
-		String query = args[1];
-		String field = "contents";
-
-		Query q = expand( query, searcher, new StandardAnalyzer(Version.LUCENE_CURRENT), field, 0.9f);
-		System.out.println( "Query: " + q.toString( field));
-
-
-
-		searcher.close();
-		directory.close();
-	}
-
-
-	/**
 	 * Perform synonym expansion on a query.
 	 *
 	 * @param query users query that is assumed to not have any "special" query syntax, thus it should be just normal words, so "big dog" makes sense, but a query like "title:foo^1.2" doesn't as this should presumably be passed directly to the default query parser.
 	 *
 	 * @param syns a opened to the Lucene index you previously created with {@link Syns2Index}. The searcher is not closed or otherwise altered.
 	 *
-	 * @param a optional analyzer used to parse the users query else {@link StandardAnalyzer} is used
+	 * @param a analyzer used to parse the users query.
 	 *
 	 * @param f optional field name to search in or null if you want the default of "contents"
 	 *
@@ -113,7 +74,6 @@ public final class SynExpand {
 		final Set<String> already = new HashSet<String>(); // avoid dups 
 		List<String> top = new LinkedList<String>(); // needs to be separately listed..
 		final String field = ( f == null) ? "contents" : f;
-		if ( a == null) a = new StandardAnalyzer(Version.LUCENE_CURRENT);
 
 		// [1] Parse query into separate words so that when we expand we can avoid dups
 		TokenStream ts = a.reusableTokenStream( field, new StringReader( query));

Modified: lucene/dev/branches/flexscoring/lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/SynLookup.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/SynLookup.java?rev=1143840&r1=1143839&r2=1143840&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/SynLookup.java (original)
+++ lucene/dev/branches/flexscoring/lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/SynLookup.java Thu Jul  7 14:18:02 2011
@@ -41,6 +41,7 @@ import org.apache.lucene.search.Query;
 import org.apache.lucene.search.ScoreDoc;
 import org.apache.lucene.search.Scorer;
 import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.TotalHitCountCollector;
 import org.apache.lucene.store.FSDirectory;
 
 
@@ -48,24 +49,6 @@ import org.apache.lucene.store.FSDirecto
  * Test program to look up synonyms.
  */
 public class SynLookup {
-
-  final static class CountingCollector extends Collector {
-    public int numHits = 0;
-    
-    @Override
-    public void setScorer(Scorer scorer) throws IOException {}
-    @Override
-    public void collect(int doc) throws IOException {
-      numHits++;
-    }
-
-    @Override
-    public void setNextReader(AtomicReaderContext context) {}
-    @Override
-    public boolean acceptsDocsOutOfOrder() {
-      return true;
-    }    
-  }
   
 	public static void main(String[] args) throws IOException {
 		if (args.length != 2) {
@@ -78,16 +61,16 @@ public class SynLookup {
 
 		String word = args[1];
 		Query query = new TermQuery(new Term(Syns2Index.F_WORD, word));
-		CountingCollector countingCollector = new CountingCollector();
+		TotalHitCountCollector countingCollector = new TotalHitCountCollector();
 		searcher.search(query, countingCollector);
 
-		if (countingCollector.numHits == 0) {
+		if (countingCollector.getTotalHits() == 0) {
 			System.out.println("No synonyms found for " + word);
 		} else {
 			System.out.println("Synonyms found for \"" + word + "\":");
 		}
 
-		ScoreDoc[] hits = searcher.search(query, countingCollector.numHits).scoreDocs;
+		ScoreDoc[] hits = searcher.search(query, countingCollector.getTotalHits()).scoreDocs;
 		
 		for (int i = 0; i < hits.length; i++) {
 			Document doc = searcher.doc(hits[i].doc);

Modified: lucene/dev/branches/flexscoring/lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/Syns2Index.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/Syns2Index.java?rev=1143840&r1=1143839&r2=1143840&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/Syns2Index.java (original)
+++ lucene/dev/branches/flexscoring/lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/Syns2Index.java Thu Jul  7 14:18:02 2011
@@ -22,6 +22,7 @@ import java.io.File;
 import java.io.FileInputStream;
 import java.io.InputStreamReader;
 import java.io.PrintStream;
+import java.io.Reader;
 import java.util.Iterator;
 import java.util.LinkedList;
 import java.util.List;
@@ -31,7 +32,7 @@ import java.util.TreeMap;
 import java.util.TreeSet;
 
 import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.index.IndexWriter;
@@ -90,9 +91,15 @@ public class Syns2Index
 	public static final String F_WORD = "word";
 
 	/**
-	 *
+	 * we don't actually analyze any text (only a NOT_ANALYZED field),
+	 * but analyzer can't be null, docinverter wants the offset gap!
 	 */
-    private static final Analyzer ana = new StandardAnalyzer(Version.LUCENE_CURRENT);
+    private static final Analyzer ana = new Analyzer() {
+      @Override
+      public TokenStream tokenStream(String fieldName, Reader reader) {
+        return null;
+      }
+    };
 
     /**
      * Takes arg of prolog file name and index directory.

Modified: lucene/dev/branches/flexscoring/lucene/contrib/xml-query-parser/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/contrib/xml-query-parser/build.xml?rev=1143840&r1=1143839&r2=1143840&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/contrib/xml-query-parser/build.xml (original)
+++ lucene/dev/branches/flexscoring/lucene/contrib/xml-query-parser/build.xml Thu Jul  7 14:18:02 2011
@@ -25,7 +25,7 @@
 
   <import file="../contrib-build.xml"/>
 
-  <contrib-uptodate name="queries" property="queries.uptodate" classpath.property="queries.jar"/>
+  <contrib-uptodate name="queries-contrib" contrib-src-name="queries" property="queries.uptodate" classpath.property="queries.jar"/>
   <property name="lucene.jar" location="${common.dir}/build/lucene-core-${version}.jar"/>
   <property name="servlet.jar" location="${common.dir}/lib/servlet-api-2.4.jar"/>
   <available property="servlet.jar.present" type="file" file="${servlet.jar}"/>

Modified: lucene/dev/branches/flexscoring/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsFilterBuilder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsFilterBuilder.java?rev=1143840&r1=1143839&r2=1143840&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsFilterBuilder.java (original)
+++ lucene/dev/branches/flexscoring/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsFilterBuilder.java Thu Jul  7 14:18:02 2011
@@ -67,14 +67,7 @@ public class TermsFilterBuilder implemen
       ts.reset();
 	      while (ts.incrementToken()) {
 	        termAtt.fillBytesRef();
-				if (term == null)
-				{
-					term = new Term(fieldName, new BytesRef(bytes));
-				} else
-				{
-//					 create from previous to save fieldName.intern overhead
-					term = term.createTerm(new BytesRef(bytes)); 
-				}
+				term = new Term(fieldName, new BytesRef(bytes));
 				tf.addTerm(term);
 			}
 	    ts.end();

Modified: lucene/dev/branches/flexscoring/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsQueryBuilder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsQueryBuilder.java?rev=1143840&r1=1143839&r2=1143840&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsQueryBuilder.java (original)
+++ lucene/dev/branches/flexscoring/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsQueryBuilder.java Thu Jul  7 14:18:02 2011
@@ -64,14 +64,7 @@ public class TermsQueryBuilder implement
       ts.reset();
 			while (ts.incrementToken()) {
         termAtt.fillBytesRef();
-				if (term == null)
-				{
-					term = new Term(fieldName, new BytesRef(bytes));
-				} else
-				{
-//					 create from previous to save fieldName.intern overhead
-					term = term.createTerm(new BytesRef(bytes)); 
-				}
+        term = new Term(fieldName, new BytesRef(bytes));
 				bq.add(new BooleanClause(new TermQuery(term),BooleanClause.Occur.SHOULD));
 			}
 			ts.end();

Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/document/AbstractField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/document/AbstractField.java?rev=1143840&r1=1143839&r2=1143840&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/document/AbstractField.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/document/AbstractField.java Thu Jul  7 14:18:02 2011
@@ -21,7 +21,6 @@ import org.apache.lucene.analysis.TokenS
 import org.apache.lucene.index.FieldInvertState;  // for javadocs
 import org.apache.lucene.index.values.PerDocFieldValues;
 import org.apache.lucene.index.values.ValueType;
-import org.apache.lucene.util.StringHelper; // for javadocs
 
 
 /**
@@ -59,7 +58,7 @@ public abstract class AbstractField impl
   protected AbstractField(String name, Field.Store store, Field.Index index, Field.TermVector termVector) {
     if (name == null)
       throw new NullPointerException("name cannot be null");
-    this.name = StringHelper.intern(name);        // field names are interned
+    this.name = name;
 
     this.isStored = store.isStored();
     this.isIndexed = index.isIndexed();
@@ -109,7 +108,7 @@ public abstract class AbstractField impl
     return boost;
   }
 
-  /** Returns the name of the field as an interned string.
+  /** Returns the name of the field.
    * For example "date", "title", "body", ...
    */
   public String name()    { return name; }

Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/document/Field.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/document/Field.java?rev=1143840&r1=1143839&r2=1143840&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/document/Field.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/document/Field.java Thu Jul  7 14:18:02 2011
@@ -21,7 +21,6 @@ import java.io.Reader;
 
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.util.StringHelper;
 
 /**
   A field is a section of a Document.  Each field has two parts, a name and a
@@ -367,28 +366,6 @@ public final class Field extends Abstrac
    * </ul> 
    */ 
   public Field(String name, String value, Store store, Index index, TermVector termVector) {
-    this(name, true, value, store, index, termVector);
-  }
-  
-  /**
-   * Create a field by specifying its name, value and how it will
-   * be saved in the index.
-   * 
-   * @param name The name of the field
-   * @param internName Whether to .intern() name or not
-   * @param value The string to process
-   * @param store Whether <code>value</code> should be stored in the index
-   * @param index Whether the field should be indexed, and if so, if it should
-   *  be tokenized before indexing 
-   * @param termVector Whether term vector should be stored
-   * @throws NullPointerException if name or value is <code>null</code>
-   * @throws IllegalArgumentException in any of the following situations:
-   * <ul> 
-   *  <li>the field is neither stored nor indexed</li> 
-   *  <li>the field is not indexed but termVector is <code>TermVector.YES</code></li>
-   * </ul> 
-   */ 
-  public Field(String name, boolean internName, String value, Store store, Index index, TermVector termVector) {
     if (name == null)
       throw new NullPointerException("name cannot be null");
     if (value == null)
@@ -402,9 +379,6 @@ public final class Field extends Abstrac
       throw new IllegalArgumentException("cannot store term vector information "
          + "for a field that is not indexed");
           
-    if (internName) // field names are optionally interned
-      name = StringHelper.intern(name);
-    
     this.name = name; 
     
     this.fieldsData = value;
@@ -422,7 +396,7 @@ public final class Field extends Abstrac
 
     setStoreTermVector(termVector);
   }
-
+  
   /**
    * Create a tokenized and indexed field that is not stored. Term vectors will
    * not be stored.  The Reader is read only when the Document is added to the index,
@@ -454,7 +428,7 @@ public final class Field extends Abstrac
     if (reader == null)
       throw new NullPointerException("reader cannot be null");
     
-    this.name = StringHelper.intern(name);        // field names are interned
+    this.name = name;
     this.fieldsData = reader;
     
     this.isStored = false;
@@ -500,7 +474,7 @@ public final class Field extends Abstrac
     if (tokenStream == null)
       throw new NullPointerException("tokenStream cannot be null");
     
-    this.name = StringHelper.intern(name);        // field names are interned
+    this.name = name;
     this.fieldsData = null;
     this.tokenStream = tokenStream;
 
@@ -540,7 +514,7 @@ public final class Field extends Abstrac
     if (value == null)
       throw new IllegalArgumentException("value cannot be null");
     
-    this.name = StringHelper.intern(name);        // field names are interned
+    this.name = name;
     fieldsData = value;
     
     isStored = true;

Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/document/Fieldable.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/document/Fieldable.java?rev=1143840&r1=1143839&r2=1143840&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/document/Fieldable.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/document/Fieldable.java Thu Jul  7 14:18:02 2011
@@ -71,7 +71,7 @@ public interface Fieldable {
    */
   float getBoost();
 
-  /** Returns the name of the field as an interned string.
+  /** Returns the name of the field.
    * For example "date", "title", "body", ...
    */
   String name();

Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/document/IndexDocValuesField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/document/IndexDocValuesField.java?rev=1143840&r1=1143839&r2=1143840&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/document/IndexDocValuesField.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/document/IndexDocValuesField.java Thu Jul  7 14:18:02 2011
@@ -98,12 +98,99 @@ public class IndexDocValuesField extends
 
   /**
    * Sets the given <code>long</code> value and sets the field's {@link ValueType} to
-   * {@link ValueType#INTS} unless already set. If you want to change the
+   * {@link ValueType#VAR_INTS} unless already set. If you want to change the
    * default type use {@link #setType(ValueType)}.
    */
   public void setInt(long value) {
+    setInt(value, false);
+  }
+  
+  /**
+   * Sets the given <code>long</code> value as a 64 bit signed integer.
+   * 
+   * @param value
+   *          the value to set
+   * @param fixed
+   *          if <code>true</code> {@link ValueType#FIXED_INTS_64} is used
+   *          otherwise {@link ValueType#VAR_INTS}
+   */
+  public void setInt(long value, boolean fixed) {
+    if (type == null) {
+      type = fixed ? ValueType.FIXED_INTS_64 : ValueType.VAR_INTS;
+    }
+    longValue = value;
+  }
+
+  /**
+   * Sets the given <code>int</code> value and sets the field's {@link ValueType} to
+   * {@link ValueType#VAR_INTS} unless already set. If you want to change the
+   * default type use {@link #setType(ValueType)}.
+   */
+  public void setInt(int value) {
+    setInt(value, false);
+  }
+
+  /**
+   * Sets the given <code>int</code> value as a 32 bit signed integer.
+   * 
+   * @param value
+   *          the value to set
+   * @param fixed
+   *          if <code>true</code> {@link ValueType#FIXED_INTS_32} is used
+   *          otherwise {@link ValueType#VAR_INTS}
+   */
+  public void setInt(int value, boolean fixed) {
+    if (type == null) {
+      type = fixed ? ValueType.FIXED_INTS_32 : ValueType.VAR_INTS;
+    }
+    longValue = value;
+  }
+
+  /**
+   * Sets the given <code>short</code> value and sets the field's {@link ValueType} to
+   * {@link ValueType#VAR_INTS} unless already set. If you want to change the
+   * default type use {@link #setType(ValueType)}.
+   */
+  public void setInt(short value) {
+    setInt(value, false);
+  }
+
+  /**
+   * Sets the given <code>short</code> value as a 16 bit signed integer.
+   * 
+   * @param value
+   *          the value to set
+   * @param fixed
+   *          if <code>true</code> {@link ValueType#FIXED_INTS_16} is used
+   *          otherwise {@link ValueType#VAR_INTS}
+   */
+  public void setInt(short value, boolean fixed) {
+    if (type == null) {
+      type = fixed ? ValueType.FIXED_INTS_16 : ValueType.VAR_INTS;
+    }
+    longValue = value;
+  }
+
+  /**
+   * Sets the given <code>byte</code> value and sets the field's {@link ValueType} to
+   * {@link ValueType#VAR_INTS} unless already set. If you want to change the
+   * default type use {@link #setType(ValueType)}.
+   */
+  public void setInt(byte value) {
+    setInt(value, false);
+  }
+  /**
+   * Sets the given <code>byte</code> value as a 8 bit signed integer.
+   * 
+   * @param value
+   *          the value to set
+   * @param fixed
+   *          if <code>true</code> {@link ValueType#FIXED_INTS_8} is used
+   *          otherwise {@link ValueType#VAR_INTS}
+   */
+  public void setInt(byte value, boolean fixed) {
     if (type == null) {
-      type = ValueType.INTS;
+      type = fixed ? ValueType.FIXED_INTS_8 : ValueType.VAR_INTS;
     }
     longValue = value;
   }
@@ -268,7 +355,7 @@ public class IndexDocValuesField extends
           field.stringValue());
       valField.setBytes(ref, type);
       break;
-    case INTS:
+    case VAR_INTS:
       valField.setInt(Long.parseLong(field.stringValue()));
       break;
     case FLOAT_32:

Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/document/NumericField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/document/NumericField.java?rev=1143840&r1=1143839&r2=1143840&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/document/NumericField.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/document/NumericField.java Thu Jul  7 14:18:02 2011
@@ -24,7 +24,6 @@ import org.apache.lucene.analysis.Numeri
 import org.apache.lucene.util.NumericUtils;
 import org.apache.lucene.search.NumericRangeQuery; // javadocs
 import org.apache.lucene.search.NumericRangeFilter; // javadocs
-import org.apache.lucene.search.SortField; // javadocs
 import org.apache.lucene.search.FieldCache; // javadocs
 
 /**
@@ -69,8 +68,8 @@ import org.apache.lucene.search.FieldCac
  * <code>NumericField</code>, use {@link NumericRangeQuery} or {@link
  * NumericRangeFilter}.  To sort according to a
  * <code>NumericField</code>, use the normal numeric sort types, eg
- * {@link SortField#INT}. <code>NumericField</code> values
- * can also be loaded directly from {@link FieldCache}.</p>
+ * {@link org.apache.lucene.search.SortField.Type#INT}. <code>NumericField</code> 
+ * values can also be loaded directly from {@link FieldCache}.</p>
  *
  * <p>By default, a <code>NumericField</code>'s value is not stored but
  * is indexed for range filtering and sorting.  You can use

Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/BufferedDeletes.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/BufferedDeletes.java?rev=1143840&r1=1143839&r2=1143840&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/BufferedDeletes.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/BufferedDeletes.java Thu Jul  7 14:18:02 2011
@@ -47,11 +47,12 @@ class BufferedDeletes {
      key, Integer val, int hash, Entry next
      (OBJ_HEADER + 3*POINTER + INT).  Term is object w/
      String field and String text (OBJ_HEADER + 2*POINTER).
-     We don't count Term's field since it's interned.
+     Term's field is String (OBJ_HEADER + 4*INT + POINTER +
+     OBJ_HEADER + string.length*CHAR).
      Term's text is String (OBJ_HEADER + 4*INT + POINTER +
      OBJ_HEADER + string.length*CHAR).  Integer is
      OBJ_HEADER + INT. */
-  final static int BYTES_PER_DEL_TERM = 8*RamUsageEstimator.NUM_BYTES_OBJECT_REF + 5*RamUsageEstimator.NUM_BYTES_OBJECT_HEADER + 6*RamUsageEstimator.NUM_BYTES_INT;
+  final static int BYTES_PER_DEL_TERM = 9*RamUsageEstimator.NUM_BYTES_OBJECT_REF + 7*RamUsageEstimator.NUM_BYTES_OBJECT_HEADER + 10*RamUsageEstimator.NUM_BYTES_INT;
 
   /* Rough logic: del docIDs are List<Integer>.  Say list
      allocates ~2X size (2*POINTER).  Integer is OBJ_HEADER
@@ -189,13 +190,13 @@ class BufferedDeletes {
     terms.put(term, Integer.valueOf(docIDUpto));
     numTermDeletes.incrementAndGet();
     if (current == null) {
-      bytesUsed.addAndGet(BYTES_PER_DEL_TERM + term.bytes.length);
+      bytesUsed.addAndGet(BYTES_PER_DEL_TERM + term.bytes.length + (RamUsageEstimator.NUM_BYTES_CHAR * term.field().length()));
     }
   }
 
   public Iterable<Term> termsIterable() {
     return new Iterable<Term>() {
-      // @Override -- not until Java 1.6
+      @Override
       public Iterator<Term> iterator() {
         return terms.keySet().iterator();
       }
@@ -205,23 +206,23 @@ class BufferedDeletes {
   public Iterable<QueryAndLimit> queriesIterable() {
     return new Iterable<QueryAndLimit>() {
       
-      // @Override -- not until Java 1.6
+      @Override
       public Iterator<QueryAndLimit> iterator() {
         return new Iterator<QueryAndLimit>() {
           private final Iterator<Map.Entry<Query,Integer>> iter = queries.entrySet().iterator();
 
-          // @Override -- not until Java 1.6
+          @Override
           public boolean hasNext() {
             return iter.hasNext();
           }
 
-          // @Override -- not until Java 1.6
+          @Override
           public QueryAndLimit next() {
             final Map.Entry<Query,Integer> ent = iter.next();
             return new QueryAndLimit(ent.getKey(), ent.getValue());
           }
 
-          // @Override -- not until Java 1.6
+          @Override
           public void remove() {
             throw new UnsupportedOperationException();
           }

Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/BufferedDeletesStream.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/BufferedDeletesStream.java?rev=1143840&r1=1143839&r2=1143840&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/BufferedDeletesStream.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/BufferedDeletesStream.java Thu Jul  7 14:18:02 2011
@@ -143,7 +143,7 @@ class BufferedDeletesStream {
 
   // Sorts SegmentInfos from smallest to biggest bufferedDelGen:
   private static final Comparator<SegmentInfo> sortSegInfoByDelGen = new Comparator<SegmentInfo>() {
-    // @Override -- not until Java 1.6
+    @Override
     public int compare(SegmentInfo si1, SegmentInfo si2) {
       final long cmp = si1.getBufferedDeletesGen() - si2.getBufferedDeletesGen();
       if (cmp > 0) {
@@ -162,7 +162,7 @@ class BufferedDeletesStream {
   };
   
   /** Resolves the buffered deleted Term/Query/docIDs, into
-   *  actual deleted docIDs in the deletedDocs BitVector for
+   *  actual deleted docIDs in the liveDocs BitVector for
    *  each SegmentReader. */
   public synchronized ApplyDeletesResult applyDeletes(IndexWriter.ReaderPool readerPool, List<SegmentInfo> infos) throws IOException {
     final long t0 = System.currentTimeMillis();
@@ -380,7 +380,7 @@ class BufferedDeletesStream {
       // Since we visit terms sorted, we gain performance
       // by re-using the same TermsEnum and seeking only
       // forwards
-      if (term.field() != currentField) {
+      if (!term.field().equals(currentField)) {
         assert currentField == null || currentField.compareTo(term.field()) < 0;
         currentField = term.field();
         Terms terms = fields.terms(currentField);
@@ -398,8 +398,8 @@ class BufferedDeletesStream {
 
       // System.out.println("  term=" + term);
 
-      if (termsEnum.seek(term.bytes(), false) == TermsEnum.SeekStatus.FOUND) {
-        DocsEnum docsEnum = termsEnum.docs(reader.getDeletedDocs(), docs);
+      if (termsEnum.seekExact(term.bytes(), false)) {
+        DocsEnum docsEnum = termsEnum.docs(reader.getLiveDocs(), docs);
 
         if (docsEnum != null) {
           while (true) {

Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/CheckIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/CheckIndex.java?rev=1143840&r1=1143839&r2=1143840&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/CheckIndex.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/CheckIndex.java Thu Jul  7 14:18:02 2011
@@ -17,12 +17,16 @@ package org.apache.lucene.index;
  * limitations under the License.
  */
 
-import org.apache.lucene.search.DocIdSetIterator;
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.TermQuery;
-import org.apache.lucene.store.FSDirectory;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.IndexInput;
+import java.io.File;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.text.NumberFormat;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Map;
+
 import org.apache.lucene.document.AbstractField;  // for javadocs
 import org.apache.lucene.document.Document;
 import org.apache.lucene.index.codecs.CodecProvider;
@@ -30,18 +34,15 @@ import org.apache.lucene.index.codecs.De
 import org.apache.lucene.index.codecs.PerDocValues;
 import org.apache.lucene.index.values.IndexDocValues;
 import org.apache.lucene.index.values.ValuesEnum;
+import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FSDirectory;
+import org.apache.lucene.store.IndexInput;
 import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.BytesRef;
-
-import java.text.NumberFormat;
-import java.io.PrintStream;
-import java.io.IOException;
-import java.io.File;
-import java.util.Collection;
-import java.util.Comparator;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
+import org.apache.lucene.util.StringHelper;
 
 /**
  * Basic tool and API to check the health of an index and
@@ -338,6 +339,27 @@ public class CheckIndex {
       return result;
     }
 
+    // find the oldest and newest segment versions
+    String oldest = Integer.toString(Integer.MAX_VALUE), newest = Integer.toString(Integer.MIN_VALUE);
+    String oldSegs = null;
+    boolean foundNonNullVersion = false;
+    Comparator<String> versionComparator = StringHelper.getVersionComparator();
+    for (SegmentInfo si : sis) {
+      String version = si.getVersion();
+      if (version == null) {
+        // pre-3.1 segment
+        oldSegs = "pre-3.1";
+      } else {
+        foundNonNullVersion = true;
+        if (versionComparator.compare(version, oldest) < 0) {
+          oldest = version;
+        }
+        if (versionComparator.compare(version, newest) > 0) {
+          newest = version;
+        }
+      }
+    }
+
     final int numSegments = sis.size();
     final String segmentsFileName = sis.getCurrentSegmentFileName();
     IndexInput input = null;
@@ -372,7 +394,7 @@ public class CheckIndex {
     } else if (format == DefaultSegmentInfosWriter.FORMAT_HAS_VECTORS) {
       sFormat = "FORMAT_HAS_VECTORS [Lucene 3.1]";
     } else if (format == DefaultSegmentInfosWriter.FORMAT_3_1) {
-      sFormat = "FORMAT_3_1 [Lucene 3.1]";
+      sFormat = "FORMAT_3_1 [Lucene 3.1+]";
     } else if (format == DefaultSegmentInfosWriter.FORMAT_4_0) {
       sFormat = "FORMAT_4_0 [Lucene 4.0]";
     } else if (format == DefaultSegmentInfosWriter.FORMAT_CURRENT) {
@@ -396,7 +418,19 @@ public class CheckIndex {
       userDataString = "";
     }
 
-    msg("Segments file=" + segmentsFileName + " numSegments=" + numSegments + " version=" + sFormat + userDataString);
+    String versionString = null;
+    if (oldSegs != null) {
+      if (foundNonNullVersion) {
+        versionString = "versions=[" + oldSegs + " .. " + newest + "]";
+      } else {
+        versionString = "version=" + oldSegs;
+      }
+    } else {
+      versionString = oldest.equals(newest) ? ( "version=" + oldest ) : ("versions=[" + oldest + " .. " + newest + "]");
+    }
+
+    msg("Segments file=" + segmentsFileName + " numSegments=" + numSegments
+        + " " + versionString + " format=" + sFormat + userDataString);
 
     if (onlySegments != null) {
       result.partial = true;
@@ -486,13 +520,13 @@ public class CheckIndex {
         final int numDocs = reader.numDocs();
         toLoseDocCount = numDocs;
         if (reader.hasDeletions()) {
-          if (reader.deletedDocs.count() != info.getDelCount()) {
-            throw new RuntimeException("delete count mismatch: info=" + info.getDelCount() + " vs deletedDocs.count()=" + reader.deletedDocs.count());
+          if (reader.liveDocs.count() != info.docCount - info.getDelCount()) {
+            throw new RuntimeException("delete count mismatch: info=" + (info.docCount - info.getDelCount()) + " vs reader=" + reader.liveDocs.count());
           }
-          if (reader.deletedDocs.count() > reader.maxDoc()) {
-            throw new RuntimeException("too many deleted docs: maxDoc()=" + reader.maxDoc() + " vs deletedDocs.count()=" + reader.deletedDocs.count());
+          if ((info.docCount-reader.liveDocs.count()) > reader.maxDoc()) {
+            throw new RuntimeException("too many deleted docs: maxDoc()=" + reader.maxDoc() + " vs del count=" + (info.docCount-reader.liveDocs.count()));
           }
-          if (info.docCount - numDocs != info.getDelCount()){
+          if (info.docCount - numDocs != info.getDelCount()) {
             throw new RuntimeException("delete count mismatch: info=" + info.getDelCount() + " vs reader=" + (info.docCount - numDocs));
           }
           segInfoStat.numDeleted = info.docCount - numDocs;
@@ -620,7 +654,7 @@ public class CheckIndex {
     final Status.TermIndexStatus status = new Status.TermIndexStatus();
 
     final int maxDoc = reader.maxDoc();
-    final Bits delDocs = reader.getDeletedDocs();
+    final Bits liveDocs = reader.getLiveDocs();
 
     final IndexSearcher is = new IndexSearcher(reader);
 
@@ -678,8 +712,8 @@ public class CheckIndex {
           final int docFreq = terms.docFreq();
           status.totFreq += docFreq;
 
-          docs = terms.docs(delDocs, docs);
-          postings = terms.docsAndPositions(delDocs, postings);
+          docs = terms.docs(liveDocs, docs);
+          postings = terms.docsAndPositions(liveDocs, postings);
 
           if (hasOrd) {
             long ord = -1;
@@ -781,7 +815,7 @@ public class CheckIndex {
             if (hasPositions) {
               for(int idx=0;idx<7;idx++) {
                 final int skipDocID = (int) (((idx+1)*(long) maxDoc)/8);
-                postings = terms.docsAndPositions(delDocs, postings);
+                postings = terms.docsAndPositions(liveDocs, postings);
                 final int docID = postings.advance(skipDocID);
                 if (docID == DocsEnum.NO_MORE_DOCS) {
                   break;
@@ -817,7 +851,7 @@ public class CheckIndex {
             } else {
               for(int idx=0;idx<7;idx++) {
                 final int skipDocID = (int) (((idx+1)*(long) maxDoc)/8);
-                docs = terms.docs(delDocs, docs);
+                docs = terms.docs(liveDocs, docs);
                 final int docID = docs.advance(skipDocID);
                 if (docID == DocsEnum.NO_MORE_DOCS) {
                   break;
@@ -847,7 +881,7 @@ public class CheckIndex {
 
         // Test seek to last term:
         if (lastTerm != null) {
-          if (terms.seek(lastTerm) != TermsEnum.SeekStatus.FOUND) {
+          if (terms.seekCeil(lastTerm) != TermsEnum.SeekStatus.FOUND) {
             throw new RuntimeException("seek to last term " + lastTerm + " failed");
           }
 
@@ -874,18 +908,18 @@ public class CheckIndex {
             // Seek by ord
             for(int i=seekCount-1;i>=0;i--) {
               long ord = i*(termCount/seekCount);
-              terms.seek(ord);
+              terms.seekExact(ord);
               seekTerms[i] = new BytesRef(terms.term());
             }
 
             // Seek by term
             long totDocCount = 0;
             for(int i=seekCount-1;i>=0;i--) {
-              if (terms.seek(seekTerms[i]) != TermsEnum.SeekStatus.FOUND) {
+              if (terms.seekCeil(seekTerms[i]) != TermsEnum.SeekStatus.FOUND) {
                 throw new RuntimeException("seek to existing term " + seekTerms[i] + " failed");
               }
               
-              docs = terms.docs(delDocs, docs);
+              docs = terms.docs(liveDocs, docs);
               if (docs == null) {
                 throw new RuntimeException("null DocsEnum from to existing term " + seekTerms[i]);
               }
@@ -933,9 +967,9 @@ public class CheckIndex {
       }
 
       // Scan stored fields for all documents
-      final Bits delDocs = reader.getDeletedDocs();
+      final Bits liveDocs = reader.getLiveDocs();
       for (int j = 0; j < info.docCount; ++j) {
-        if (delDocs == null || !delDocs.get(j)) {
+        if (liveDocs == null || liveDocs.get(j)) {
           status.docCount++;
           Document doc = reader.document(j);
           status.totFields += doc.getFields().size();
@@ -991,7 +1025,11 @@ public class CheckIndex {
             case FLOAT_64:
               values.getFloat();
               break;
-            case INTS:
+            case VAR_INTS:
+            case FIXED_INTS_16:
+            case FIXED_INTS_32:
+            case FIXED_INTS_64:
+            case FIXED_INTS_8:
               values.getInt();
               break;
             default:
@@ -1025,9 +1063,9 @@ public class CheckIndex {
         infoStream.print("    test: term vectors........");
       }
 
-      final Bits delDocs = reader.getDeletedDocs();
+      final Bits liveDocs = reader.getLiveDocs();
       for (int j = 0; j < info.docCount; ++j) {
-        if (delDocs == null || !delDocs.get(j)) {
+        if (liveDocs == null || liveDocs.get(j)) {
           status.docCount++;
           TermFreqVector[] tfv = reader.getTermFreqVectors(j);
           if (tfv != null) {

Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/DirectoryReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/DirectoryReader.java?rev=1143840&r1=1143839&r2=1143840&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/DirectoryReader.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/DirectoryReader.java Thu Jul  7 14:18:02 2011
@@ -350,8 +350,8 @@ class DirectoryReader extends IndexReade
   }
 
   @Override
-  public Bits getDeletedDocs() {
-    throw new UnsupportedOperationException("please use MultiFields.getDeletedDocs, or wrap your IndexReader with SlowMultiReaderWrapper, if you really need a top level Bits deletedDocs");
+  public Bits getLiveDocs() {
+    throw new UnsupportedOperationException("please use MultiFields.getLiveDocs, or wrap your IndexReader with SlowMultiReaderWrapper, if you really need a top level Bits liveDocs");
   }
 
   @Override

Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/DocTermOrds.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/DocTermOrds.java?rev=1143840&r1=1143839&r2=1143840&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/DocTermOrds.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/DocTermOrds.java Thu Jul  7 14:18:02 2011
@@ -237,7 +237,7 @@ public class DocTermOrds {
     final TermsEnum te = terms.iterator();
     final BytesRef seekStart = termPrefix != null ? termPrefix : new BytesRef();
     //System.out.println("seekStart=" + seekStart.utf8ToString());
-    if (te.seek(seekStart) == TermsEnum.SeekStatus.END) {
+    if (te.seekCeil(seekStart) == TermsEnum.SeekStatus.END) {
       // No terms match
       return;
     }
@@ -249,7 +249,7 @@ public class DocTermOrds {
 
     boolean testedOrd = false;
 
-    final Bits delDocs = MultiFields.getDeletedDocs(reader);
+    final Bits liveDocs = MultiFields.getLiveDocs(reader);
 
     // we need a minimum of 9 bytes, but round up to 12 since the space would
     // be wasted with most allocators anyway.
@@ -312,7 +312,7 @@ public class DocTermOrds {
       final int df = te.docFreq();
       if (df <= maxTermDocFreq) {
 
-        docsEnum = te.docs(delDocs, docsEnum);
+        docsEnum = te.docs(liveDocs, docsEnum);
 
         final DocsEnum.BulkReadResult bulkResult = docsEnum.getBulkResult();
 
@@ -653,13 +653,13 @@ public class DocTermOrds {
     }
 
     @Override    
-    public DocsEnum docs(Bits skipDocs, DocsEnum reuse) throws IOException {
-      return termsEnum.docs(skipDocs, reuse);
+    public DocsEnum docs(Bits liveDocs, DocsEnum reuse) throws IOException {
+      return termsEnum.docs(liveDocs, reuse);
     }
 
     @Override    
-    public DocsAndPositionsEnum docsAndPositions(Bits skipDocs, DocsAndPositionsEnum reuse) throws IOException {
-      return termsEnum.docsAndPositions(skipDocs, reuse);
+    public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse) throws IOException {
+      return termsEnum.docsAndPositions(liveDocs, reuse);
     }
 
     @Override
@@ -693,7 +693,7 @@ public class DocTermOrds {
     }
 
     @Override
-    public SeekStatus seek(BytesRef target, boolean useCache) throws IOException {
+    public SeekStatus seekCeil(BytesRef target, boolean useCache) throws IOException {
 
       // already here
       if (term != null && term.equals(target)) {
@@ -704,7 +704,7 @@ public class DocTermOrds {
 
       if (startIdx >= 0) {
         // we hit the term exactly... lucky us!
-        TermsEnum.SeekStatus seekStatus = termsEnum.seek(target);
+        TermsEnum.SeekStatus seekStatus = termsEnum.seekCeil(target);
         assert seekStatus == TermsEnum.SeekStatus.FOUND;
         ord = startIdx << indexIntervalBits;
         setTerm();
@@ -717,7 +717,7 @@ public class DocTermOrds {
     
       if (startIdx == 0) {
         // our target occurs *before* the first term
-        TermsEnum.SeekStatus seekStatus = termsEnum.seek(target);
+        TermsEnum.SeekStatus seekStatus = termsEnum.seekCeil(target);
         assert seekStatus == TermsEnum.SeekStatus.NOT_FOUND;
         ord = 0;
         setTerm();
@@ -733,7 +733,7 @@ public class DocTermOrds {
         // so we don't need to seek.
       } else {
         // seek to the right block
-        TermsEnum.SeekStatus seekStatus = termsEnum.seek(indexedTermsArray[startIdx]);
+        TermsEnum.SeekStatus seekStatus = termsEnum.seekCeil(indexedTermsArray[startIdx]);
         assert seekStatus == TermsEnum.SeekStatus.FOUND;
         ord = startIdx << indexIntervalBits;
         setTerm();
@@ -754,16 +754,16 @@ public class DocTermOrds {
     }
 
     @Override
-    public SeekStatus seek(long targetOrd) throws IOException {
+    public void seekExact(long targetOrd) throws IOException {
       int delta = (int) (targetOrd - ordBase - ord);
-      //System.out.println("  seek(ord) targetOrd=" + targetOrd + " delta=" + delta + " ord=" + ord);
+      //System.out.println("  seek(ord) targetOrd=" + targetOrd + " delta=" + delta + " ord=" + ord + " ii=" + indexInterval);
       if (delta < 0 || delta > indexInterval) {
         final int idx = (int) (targetOrd >>> indexIntervalBits);
         final BytesRef base = indexedTermsArray[idx];
         //System.out.println("  do seek term=" + base.utf8ToString());
         ord = idx << indexIntervalBits;
         delta = (int) (targetOrd - ord);
-        final TermsEnum.SeekStatus seekStatus = termsEnum.seek(base, true);
+        final TermsEnum.SeekStatus seekStatus = termsEnum.seekCeil(base, true);
         assert seekStatus == TermsEnum.SeekStatus.FOUND;
       } else {
         //System.out.println("seek w/in block");
@@ -772,15 +772,14 @@ public class DocTermOrds {
       while (--delta >= 0) {
         BytesRef br = termsEnum.next();
         if (br == null) {
-          term = null;
-          return null;
+          assert false;
+          return;
         }
         ord++;
       }
 
       setTerm();
-      return term == null ? SeekStatus.END : SeekStatus.FOUND;
-      //System.out.println("  return term=" + term.utf8ToString());
+      assert term != null;
     }
 
     private BytesRef setTerm() throws IOException {
@@ -794,8 +793,7 @@ public class DocTermOrds {
   }
 
   public BytesRef lookupTerm(TermsEnum termsEnum, int ord) throws IOException {
-    TermsEnum.SeekStatus status = termsEnum.seek(ord);
-    assert status == TermsEnum.SeekStatus.FOUND;
+    termsEnum.seekExact(ord);
     return termsEnum.term();
   }
 }

Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/DocumentsWriterFlushControl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/DocumentsWriterFlushControl.java?rev=1143840&r1=1143839&r2=1143840&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/DocumentsWriterFlushControl.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/DocumentsWriterFlushControl.java Thu Jul  7 14:18:02 2011
@@ -99,7 +99,7 @@ public final class DocumentsWriterFlushC
       final long ram = flushBytes + activeBytes;
       final long ramBufferBytes = (long) (maxConfiguredRamBuffer * 1024 * 1024);
       // take peakDelta into account - worst case is that all flushing, pending and blocked DWPT had maxMem and the last doc had the peakDelta 
-      final long expected = (long)(2 * (ramBufferBytes)) + ((numPending + numFlushingDWPT() + numBlockedFlushes()) * peakDelta);
+      final long expected = (2 * (ramBufferBytes)) + ((numPending + numFlushingDWPT() + numBlockedFlushes()) * peakDelta);
       if (peakDelta < (ramBufferBytes >> 1)) {
         /*
          * if we are indexing with very low maxRamBuffer like 0.1MB memory can

Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java?rev=1143840&r1=1143839&r2=1143840&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java Thu Jul  7 14:18:02 2011
@@ -111,13 +111,13 @@ public class DocumentsWriterPerThread {
   static class FlushedSegment {
     final SegmentInfo segmentInfo;
     final BufferedDeletes segmentDeletes;
-    final BitVector deletedDocuments;
+    final BitVector liveDocs;
 
     private FlushedSegment(SegmentInfo segmentInfo,
-        BufferedDeletes segmentDeletes, BitVector deletedDocuments) {
+        BufferedDeletes segmentDeletes, BitVector liveDocs) {
       this.segmentInfo = segmentInfo;
       this.segmentDeletes = segmentDeletes;
-      this.deletedDocuments = deletedDocuments;
+      this.liveDocs = liveDocs;
     }
   }
 
@@ -434,9 +434,10 @@ public class DocumentsWriterPerThread {
     // happens when an exception is hit processing that
     // doc, eg if analyzer has some problem w/ the text):
     if (pendingDeletes.docIDs.size() > 0) {
-      flushState.deletedDocs = new BitVector(numDocsInRAM);
+      flushState.liveDocs = new BitVector(numDocsInRAM);
+      flushState.liveDocs.invertAll();
       for(int delDocID : pendingDeletes.docIDs) {
-        flushState.deletedDocs.set(delDocID);
+        flushState.liveDocs.clear(delDocID);
       }
       pendingDeletes.bytesUsed.addAndGet(-pendingDeletes.docIDs.size() * BufferedDeletes.BYTES_PER_DEL_DOCID);
       pendingDeletes.docIDs.clear();
@@ -460,7 +461,7 @@ public class DocumentsWriterPerThread {
       pendingDeletes.terms.clear();
       final SegmentInfo newSegment = new SegmentInfo(segment, flushState.numDocs, directory, false, flushState.segmentCodecs, fieldInfos.asReadOnly());
       if (infoStream != null) {
-        message("new segment has " + (flushState.deletedDocs == null ? 0 : flushState.deletedDocs.count()) + " deleted docs");
+        message("new segment has " + (flushState.liveDocs == null ? 0 : (flushState.numDocs - flushState.liveDocs.count())) + " deleted docs");
         message("new segment has " + (newSegment.getHasVectors() ? "vectors" : "no vectors"));
         message("flushedFiles=" + newSegment.files());
         message("flushed codecs=" + newSegment.getSegmentCodecs());
@@ -489,7 +490,7 @@ public class DocumentsWriterPerThread {
       doAfterFlush();
       success = true;
 
-      return new FlushedSegment(newSegment, segmentDeletes, flushState.deletedDocs);
+      return new FlushedSegment(newSegment, segmentDeletes, flushState.liveDocs);
     } finally {
       if (!success) {
         if (segment != null) {

Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/FieldInfos.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/FieldInfos.java?rev=1143840&r1=1143839&r2=1143840&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/FieldInfos.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/FieldInfos.java Thu Jul  7 14:18:02 2011
@@ -36,7 +36,6 @@ import org.apache.lucene.store.Directory
 import org.apache.lucene.store.IndexInput;
 import org.apache.lucene.store.IndexOutput;
 import org.apache.lucene.util.CodecUtil;
-import org.apache.lucene.util.StringHelper;
 
 /** Access to the Fieldable Info file that describes document fields and whether or
  *  not they are indexed. Each segment has a separate Fieldable Info file. Objects
@@ -475,12 +474,11 @@ public final class FieldInfos implements
                                 boolean storeTermVector, boolean storePositionWithTermVector, 
                                 boolean storeOffsetWithTermVector, boolean omitNorms, boolean storePayloads, boolean omitTermFreqAndPositions, ValueType docValuesType) {
     // don't check modifiable here since we use that to initially build up FIs
-    name = StringHelper.intern(name);
     if (globalFieldNumbers != null) {
       globalFieldNumbers.setIfNotSet(fieldNumber, name);
     } 
     final FieldInfo fi = new FieldInfo(name, isIndexed, fieldNumber, storeTermVector, storePositionWithTermVector,
-                                 storeOffsetWithTermVector, omitNorms, storePayloads, omitTermFreqAndPositions, docValuesType);
+                                       storeOffsetWithTermVector, omitNorms, storePayloads, omitTermFreqAndPositions, docValuesType);
     putInternal(fi);
     return fi;
   }
@@ -611,7 +609,7 @@ public final class FieldInfos implements
         b = 0;
       } else {
         switch(fi.docValues) {
-        case INTS:
+        case VAR_INTS:
           b = 1;
           break;
         case FLOAT_32:
@@ -638,6 +636,19 @@ public final class FieldInfos implements
         case BYTES_VAR_SORTED:
           b = 9;
           break;
+        case FIXED_INTS_16:
+          b = 10;
+          break;
+        case FIXED_INTS_32:
+          b = 11;
+          break;
+        case FIXED_INTS_64:
+          b = 12;
+          break;
+        case FIXED_INTS_8:
+          b = 13;
+          break;
+       
         default:
           throw new IllegalStateException("unhandled indexValues type " + fi.docValues);
         }
@@ -659,7 +670,7 @@ public final class FieldInfos implements
     final int size = input.readVInt(); //read in the size
 
     for (int i = 0; i < size; i++) {
-      String name = StringHelper.intern(input.readString());
+      String name = input.readString();
       // if this is a previous format codec 0 will be preflex!
       final int fieldNumber = format <= FORMAT_PER_FIELD_CODEC? input.readInt():i;
       final int codecId = format <= FORMAT_PER_FIELD_CODEC? input.readInt():0;
@@ -688,7 +699,7 @@ public final class FieldInfos implements
           docValuesType = null;
           break;
         case 1:
-          docValuesType = ValueType.INTS;
+          docValuesType = ValueType.VAR_INTS;
           break;
         case 2:
           docValuesType = ValueType.FLOAT_32;
@@ -714,6 +725,19 @@ public final class FieldInfos implements
         case 9:
           docValuesType = ValueType.BYTES_VAR_SORTED;
           break;
+        case 10:
+          docValuesType = ValueType.FIXED_INTS_16;
+          break;
+        case 11:
+          docValuesType = ValueType.FIXED_INTS_32;
+          break;
+        case 12:
+          docValuesType = ValueType.FIXED_INTS_64;
+          break;
+        case 13:
+          docValuesType = ValueType.FIXED_INTS_8;
+          break;  
+        
         default:
           throw new IllegalStateException("unhandled indexValues type " + b);
         }

Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/FieldsEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/FieldsEnum.java?rev=1143840&r1=1143839&r2=1143840&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/FieldsEnum.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/FieldsEnum.java Thu Jul  7 14:18:02 2011
@@ -43,10 +43,8 @@ public abstract class FieldsEnum {
     return atts;
   }
   
-  /** Increments the enumeration to the next field.  The
-   *  returned field is always interned, so simple ==
-   *  comparison is allowed.  Returns null when there are no
-   *  more fields.*/
+  /** Increments the enumeration to the next field. Returns
+   * null when there are no more fields.*/
   public abstract String next() throws IOException;
 
   /** Get {@link TermsEnum} for the current field.  You

Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/FieldsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/FieldsReader.java?rev=1143840&r1=1143839&r2=1143840&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/FieldsReader.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/FieldsReader.java Thu Jul  7 14:18:02 2011
@@ -364,7 +364,6 @@ public final class FieldsReader implemen
       Field.Index index = Field.Index.toIndex(fi.isIndexed, tokenize);
       Field.TermVector termVector = Field.TermVector.toTermVector(fi.storeTermVector, fi.storeOffsetWithTermVector, fi.storePositionWithTermVector);
       f = new Field(fi.name,     // name
-        false,
         fieldsStream.readString(), // read value
         Field.Store.YES,
         index,

Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/FilterIndexReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/FilterIndexReader.java?rev=1143840&r1=1143839&r2=1143840&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/FilterIndexReader.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/FilterIndexReader.java Thu Jul  7 14:18:02 2011
@@ -87,13 +87,13 @@ public class FilterIndexReader extends I
     }
 
     @Override
-    public DocsEnum docs(Bits skipDocs, BytesRef text, DocsEnum reuse) throws IOException {
-      return in.docs(skipDocs, text, reuse);
+    public DocsEnum docs(Bits liveDocs, BytesRef text, DocsEnum reuse) throws IOException {
+      return in.docs(liveDocs, text, reuse);
     }
 
     @Override
-    public DocsAndPositionsEnum docsAndPositions(Bits skipDocs, BytesRef text, DocsAndPositionsEnum reuse) throws IOException {
-      return in.docsAndPositions(skipDocs, text, reuse);
+    public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, BytesRef text, DocsAndPositionsEnum reuse) throws IOException {
+      return in.docsAndPositions(liveDocs, text, reuse);
     }
 
     @Override
@@ -132,13 +132,18 @@ public class FilterIndexReader extends I
     public FilterTermsEnum(TermsEnum in) { this.in = in; }
 
     @Override
-    public SeekStatus seek(BytesRef text, boolean useCache) throws IOException {
-      return in.seek(text, useCache);
+    public boolean seekExact(BytesRef text, boolean useCache) throws IOException {
+      return in.seekExact(text, useCache);
     }
 
     @Override
-    public SeekStatus seek(long ord) throws IOException {
-      return in.seek(ord);
+    public SeekStatus seekCeil(BytesRef text, boolean useCache) throws IOException {
+      return in.seekCeil(text, useCache);
+    }
+
+    @Override
+    public void seekExact(long ord) throws IOException {
+      in.seekExact(ord);
     }
 
     @Override
@@ -167,13 +172,13 @@ public class FilterIndexReader extends I
     }
 
     @Override
-    public DocsEnum docs(Bits skipDocs, DocsEnum reuse) throws IOException {
-      return in.docs(skipDocs, reuse);
+    public DocsEnum docs(Bits liveDocs, DocsEnum reuse) throws IOException {
+      return in.docs(liveDocs, reuse);
     }
 
     @Override
-    public DocsAndPositionsEnum docsAndPositions(Bits skipDocs, DocsAndPositionsEnum reuse) throws IOException {
-      return in.docsAndPositions(skipDocs, reuse);
+    public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse) throws IOException {
+      return in.docsAndPositions(liveDocs, reuse);
     }
 
     @Override
@@ -182,8 +187,8 @@ public class FilterIndexReader extends I
     }
 
     @Override
-    public void seek(BytesRef term, TermState state) throws IOException {
-      in.seek(term, state);
+    public void seekExact(BytesRef term, TermState state) throws IOException {
+      in.seekExact(term, state);
     }
 
     @Override
@@ -296,8 +301,8 @@ public class FilterIndexReader extends I
   }
   
   @Override
-  public Bits getDeletedDocs() {
-    return in.getDeletedDocs();
+  public Bits getLiveDocs() {
+    return in.getLiveDocs();
   }
   
   @Override

Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/FreqProxTermsWriterPerField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/FreqProxTermsWriterPerField.java?rev=1143840&r1=1143839&r2=1143840&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/FreqProxTermsWriterPerField.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/FreqProxTermsWriterPerField.java Thu Jul  7 14:18:02 2011
@@ -234,8 +234,6 @@ final class FreqProxTermsWriterPerField 
     final TermsConsumer termsConsumer = consumer.addField(fieldInfo);
     final Comparator<BytesRef> termComp = termsConsumer.getComparator();
 
-    final Term protoTerm = new Term(fieldName);
-
     final boolean currentFieldOmitTermFreqAndPositions = fieldInfo.omitTermFreqAndPositions;
 
     final Map<Term,Integer> segDeletes;
@@ -274,7 +272,7 @@ final class FreqProxTermsWriterPerField 
 
       final int delDocLimit;
       if (segDeletes != null) {
-        final Integer docIDUpto = segDeletes.get(protoTerm.createTerm(text));
+        final Integer docIDUpto = segDeletes.get(new Term(fieldName, text));
         if (docIDUpto != null) {
           delDocLimit = docIDUpto;
         } else {
@@ -341,10 +339,11 @@ final class FreqProxTermsWriterPerField 
           // Mark it deleted.  TODO: we could also skip
           // writing its postings; this would be
           // deterministic (just for this Term's docs).
-          if (state.deletedDocs == null) {
-            state.deletedDocs = new BitVector(state.numDocs);
+          if (state.liveDocs == null) {
+            state.liveDocs = new BitVector(state.numDocs);
+            state.liveDocs.invertAll();
           }
-          state.deletedDocs.set(docID);
+          state.liveDocs.clear(docID);
         }
 
         // Carefully copy over the prox + payload info,

Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/FrozenBufferedDeletes.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/FrozenBufferedDeletes.java?rev=1143840&r1=1143839&r2=1143840&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/FrozenBufferedDeletes.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/FrozenBufferedDeletes.java Thu Jul  7 14:18:02 2011
@@ -32,11 +32,12 @@ import org.apache.lucene.index.BufferedD
 class FrozenBufferedDeletes {
 
   /* Rough logic: Term is object w/
-     String field and String text (OBJ_HEADER + 2*POINTER).
-     We don't count Term's field since it's interned.
-     Term's text is String (OBJ_HEADER + 4*INT + POINTER +
-     OBJ_HEADER + string.length*CHAR). */
-  final static int BYTES_PER_DEL_TERM = 3*RamUsageEstimator.NUM_BYTES_OBJECT_REF + 3*RamUsageEstimator.NUM_BYTES_OBJECT_HEADER + 4*RamUsageEstimator.NUM_BYTES_INT;
+     String field and BytesRef text (OBJ_HEADER + 2*POINTER).
+     String field is (OBJ_HEADER + 4*INT +
+     POINTER + OBJ_HEADER + CHAR*field.length).
+     Term's text is BytesRef (OBJ_HEADER + 2*INT + POINTER +
+     OBJ_HEADER + bytes.length). */
+  final static int BYTES_PER_DEL_TERM = 4*RamUsageEstimator.NUM_BYTES_OBJECT_REF + 5*RamUsageEstimator.NUM_BYTES_OBJECT_HEADER + 6*RamUsageEstimator.NUM_BYTES_INT;
 
   /* Query we often undercount (say 24 bytes), plus int. */
   final static int BYTES_PER_DEL_QUERY = RamUsageEstimator.NUM_BYTES_OBJECT_REF + RamUsageEstimator.NUM_BYTES_INT + 24;
@@ -70,7 +71,13 @@ class FrozenBufferedDeletes {
       queryLimits[upto] = ent.getValue();
       upto++;
     }
-    bytesUsed = terms.length * BYTES_PER_DEL_TERM + queries.length * BYTES_PER_DEL_QUERY;
+    int termDataBytes = 0;
+    for(Map.Entry<Term,Integer> ent : deletes.terms.entrySet()) {
+      final Term term = ent.getKey();
+      termDataBytes += term.bytes().length;
+      termDataBytes += term.field().length() * RamUsageEstimator.NUM_BYTES_CHAR;
+    }
+    bytesUsed = terms.length * BYTES_PER_DEL_TERM + queries.length * BYTES_PER_DEL_QUERY + termDataBytes;
     numTermDeletes = deletes.numTermDeletes.get();
   }
   
@@ -86,22 +93,22 @@ class FrozenBufferedDeletes {
 
   public Iterable<Term> termsIterable() {
     return new Iterable<Term>() {
-      // @Override -- not until Java 1.6
+      @Override
       public Iterator<Term> iterator() {
         return new Iterator<Term>() {
           private int upto;
 
-          // @Override -- not until Java 1.6
+          @Override
           public boolean hasNext() {
             return upto < terms.length;
           }
 
-          // @Override -- not until Java 1.6
+          @Override
           public Term next() {
             return terms[upto++];
           }
 
-          // @Override -- not until Java 1.6
+          @Override
           public void remove() {
             throw new UnsupportedOperationException();
           }
@@ -112,24 +119,24 @@ class FrozenBufferedDeletes {
 
   public Iterable<QueryAndLimit> queriesIterable() {
     return new Iterable<QueryAndLimit>() {
-      // @Override -- not until Java 1.6
+      @Override
       public Iterator<QueryAndLimit> iterator() {
         return new Iterator<QueryAndLimit>() {
           private int upto;
 
-          // @Override -- not until Java 1.6
+          @Override
           public boolean hasNext() {
             return upto < queries.length;
           }
 
-          // @Override -- not until Java 1.6
+          @Override
           public QueryAndLimit next() {
             QueryAndLimit ret = new QueryAndLimit(queries[upto], queryLimits[upto]);
             upto++;
             return ret;
           }
 
-          // @Override -- not until Java 1.6
+          @Override
           public void remove() {
             throw new UnsupportedOperationException();
           }

Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/IndexFileNames.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/IndexFileNames.java?rev=1143840&r1=1143839&r2=1143840&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/IndexFileNames.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/IndexFileNames.java Thu Jul  7 14:18:02 2011
@@ -68,6 +68,9 @@ public final class IndexFileNames {
 
   /** Extension of compound file */
   public static final String COMPOUND_FILE_EXTENSION = "cfs";
+  
+  /** Extension of compound file entries */
+  public static final String COMPOUND_FILE_ENTRIES_EXTENSION = "cfe";
 
   /** Extension of compound file for doc store files*/
   public static final String COMPOUND_FILE_STORE_EXTENSION = "cfx";
@@ -93,6 +96,7 @@ public final class IndexFileNames {
    */
   public static final String INDEX_EXTENSIONS[] = new String[] {
     COMPOUND_FILE_EXTENSION,
+    COMPOUND_FILE_ENTRIES_EXTENSION,
     FIELD_INFOS_EXTENSION,
     FIELDS_INDEX_EXTENSION,
     FIELDS_EXTENSION,
@@ -244,7 +248,15 @@ public final class IndexFileNames {
     }
     return filename;
   }
-
+  
+  public static String stripExtension(String filename) {
+    int idx = filename.indexOf('.');
+    if (idx != -1) {
+      filename = filename.substring(0, idx);
+    }
+    return filename;
+  }
+  
   /**
    * Returns true if the given filename ends with the separate norms file
    * pattern: {@code SEPARATE_NORMS_EXTENSION + "[0-9]+"}.

Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/IndexReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/IndexReader.java?rev=1143840&r1=1143839&r2=1143840&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/IndexReader.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/IndexReader.java Thu Jul  7 14:18:02 2011
@@ -962,7 +962,7 @@ public abstract class IndexReader implem
    * requested document is deleted, and therefore asking for a deleted document
    * may yield unspecified results. Usually this is not required, however you
    * can test if the doc is deleted by checking the {@link
-   * Bits} returned from {@link MultiFields#getDeletedDocs}.
+   * Bits} returned from {@link MultiFields#getLiveDocs}.
    * 
    * @throws CorruptIndexException if the index is corrupt
    * @throws IOException if there is a low-level IO error
@@ -987,7 +987,7 @@ public abstract class IndexReader implem
    * requested document is deleted, and therefore asking for a deleted document
    * may yield unspecified results. Usually this is not required, however you
    * can test if the doc is deleted by checking the {@link
-   * Bits} returned from {@link MultiFields#getDeletedDocs}.
+   * Bits} returned from {@link MultiFields#getLiveDocs}.
    * 
    * @param n Get the document at the <code>n</code><sup>th</sup> position
    * @param fieldSelector The {@link FieldSelector} to use to determine what
@@ -1144,7 +1144,7 @@ public abstract class IndexReader implem
   /** Returns {@link DocsEnum} for the specified field &
    *  term.  This may return null, if either the field or
    *  term does not exist. */
-  public DocsEnum termDocsEnum(Bits skipDocs, String field, BytesRef term) throws IOException {
+  public DocsEnum termDocsEnum(Bits liveDocs, String field, BytesRef term) throws IOException {
     assert field != null;
     assert term != null;
     final Fields fields = fields();
@@ -1153,7 +1153,7 @@ public abstract class IndexReader implem
     }
     final Terms terms = fields.terms(field);
     if (terms != null) {
-      return terms.docs(skipDocs, term, null);
+      return terms.docs(liveDocs, term, null);
     } else {
       return null;
     }
@@ -1163,7 +1163,7 @@ public abstract class IndexReader implem
    *  field & term.  This may return null, if either the
    *  field or term does not exist, or, positions were not
    *  stored for this term. */
-  public DocsAndPositionsEnum termPositionsEnum(Bits skipDocs, String field, BytesRef term) throws IOException {
+  public DocsAndPositionsEnum termPositionsEnum(Bits liveDocs, String field, BytesRef term) throws IOException {
     assert field != null;
     assert term != null;
     final Fields fields = fields();
@@ -1172,7 +1172,7 @@ public abstract class IndexReader implem
     }
     final Terms terms = fields.terms(field);
     if (terms != null) {
-      return terms.docsAndPositions(skipDocs, term, null);
+      return terms.docsAndPositions(liveDocs, term, null);
     } else {
       return null;
     }
@@ -1183,7 +1183,7 @@ public abstract class IndexReader implem
    * {@link TermState}. This may return null, if either the field or the term
    * does not exists or the {@link TermState} is invalid for the underlying
    * implementation.*/
-  public DocsEnum termDocsEnum(Bits skipDocs, String field, BytesRef term, TermState state) throws IOException {
+  public DocsEnum termDocsEnum(Bits liveDocs, String field, BytesRef term, TermState state) throws IOException {
     assert state != null;
     assert field != null;
     final Fields fields = fields();
@@ -1192,7 +1192,7 @@ public abstract class IndexReader implem
     }
     final Terms terms = fields.terms(field);
     if (terms != null) {
-      return terms.docs(skipDocs, term, state, null);
+      return terms.docs(liveDocs, term, state, null);
     } else {
       return null;
     }
@@ -1203,7 +1203,7 @@ public abstract class IndexReader implem
    * {@link TermState}. This may return null, if either the field or the term
    * does not exists, the {@link TermState} is invalid for the underlying
    * implementation, or positions were not stored for this term.*/
-  public DocsAndPositionsEnum termPositionsEnum(Bits skipDocs, String field, BytesRef term, TermState state) throws IOException {
+  public DocsAndPositionsEnum termPositionsEnum(Bits liveDocs, String field, BytesRef term, TermState state) throws IOException {
     assert state != null;
     assert field != null;
     final Fields fields = fields();
@@ -1212,7 +1212,7 @@ public abstract class IndexReader implem
     }
     final Terms terms = fields.terms(field);
     if (terms != null) {
-      return terms.docsAndPositions(skipDocs, term, state, null);
+      return terms.docsAndPositions(liveDocs, term, state, null);
     } else {
       return null;
     }
@@ -1268,7 +1268,7 @@ public abstract class IndexReader implem
   public int deleteDocuments(Term term) throws StaleReaderException, CorruptIndexException, LockObtainFailedException, IOException {
     ensureOpen();
     DocsEnum docs = MultiFields.getTermDocsEnum(this,
-                                                MultiFields.getDeletedDocs(this),
+                                                MultiFields.getLiveDocs(this),
                                                 term.field(),
                                                 term.bytes());
     if (docs == null) return 0;
@@ -1393,15 +1393,17 @@ public abstract class IndexReader implem
    */
   public abstract Collection<String> getFieldNames(FieldOption fldOption);
 
-  /** Returns the {@link Bits} representing deleted docs.  A
-   *  set bit indicates the doc ID has been deleted.  This
-   *  method should return null when there are no deleted
-   *  docs.
-   *
-   *  The returned instance has been safely published for use by
-   *  multiple threads without additional synchronization.
+  /** Returns the {@link Bits} representing live (not
+   *  deleted) docs.  A set bit indicates the doc ID has not
+   *  been deleted.  If this method returns null it means
+   *  there are no deleted documents (all documents are
+   *  live).
+   *
+   *  The returned instance has been safely published for
+   *  use by multiple threads without additional
+   *  synchronization.
    * @lucene.experimental */
-  public abstract Bits getDeletedDocs();
+  public abstract Bits getLiveDocs();
 
   /**
    * Expert: return the IndexCommit that this reader has
@@ -1440,14 +1442,14 @@ public abstract class IndexReader implem
     }
 
     Directory dir = null;
-    CompoundFileReader cfr = null;
+    CompoundFileDirectory cfr = null;
 
     try {
       File file = new File(filename);
       String dirname = file.getAbsoluteFile().getParent();
       filename = file.getName();
       dir = FSDirectory.open(new File(dirname));
-      cfr = new CompoundFileReader(dir, filename);
+      cfr = dir.openCompoundInput(filename, BufferedIndexInput.BUFFER_SIZE);
 
       String [] files = cfr.listAll();
       ArrayUtil.mergeSort(files);   // sort the array of filename so that the output is more readable



Mime
View raw message