lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From rm...@apache.org
Subject svn commit: r1103112 [4/24] - in /lucene/dev/branches/flexscoring: ./ dev-tools/eclipse/ dev-tools/idea/.idea/ dev-tools/idea/lucene/contrib/ant/ dev-tools/idea/lucene/contrib/db/bdb-je/ dev-tools/idea/lucene/contrib/db/bdb/ dev-tools/idea/lucene/contr...
Date Sat, 14 May 2011 13:51:59 GMT
Modified: lucene/dev/branches/flexscoring/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQPHelper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQPHelper.java?rev=1103112&r1=1103111&r2=1103112&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQPHelper.java (original)
+++ lucene/dev/branches/flexscoring/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQPHelper.java Sat May 14 13:51:35 2011
@@ -191,7 +191,7 @@ public class TestQPHelper extends Lucene
 
   public StandardQueryParser getParser(Analyzer a) throws Exception {
     if (a == null)
-      a = new MockAnalyzer(MockTokenizer.SIMPLE, true);
+      a = new MockAnalyzer(random, MockTokenizer.SIMPLE, true);
     StandardQueryParser qp = new StandardQueryParser();
     qp.setAnalyzer(a);
 
@@ -281,7 +281,7 @@ public class TestQPHelper extends Lucene
 
   public Query getQueryDOA(String query, Analyzer a) throws Exception {
     if (a == null)
-      a = new MockAnalyzer(MockTokenizer.SIMPLE, true);
+      a = new MockAnalyzer(random, MockTokenizer.SIMPLE, true);
     StandardQueryParser qp = new StandardQueryParser();
     qp.setAnalyzer(a);
     qp.setDefaultOperator(Operator.AND);
@@ -301,7 +301,7 @@ public class TestQPHelper extends Lucene
   }
 
   public void testConstantScoreAutoRewrite() throws Exception {
-    StandardQueryParser qp = new StandardQueryParser(new MockAnalyzer(MockTokenizer.WHITESPACE, false));
+    StandardQueryParser qp = new StandardQueryParser(new MockAnalyzer(random, MockTokenizer.WHITESPACE, false));
     Query q = qp.parse("foo*bar", "field");
     assertTrue(q instanceof WildcardQuery);
     assertEquals(MultiTermQuery.CONSTANT_SCORE_AUTO_REWRITE_DEFAULT, ((MultiTermQuery) q).getRewriteMethod());
@@ -410,9 +410,9 @@ public class TestQPHelper extends Lucene
   public void testSimple() throws Exception {
     assertQueryEquals("\"term germ\"~2", null, "\"term germ\"~2");
     assertQueryEquals("term term term", null, "term term term");
-    assertQueryEquals("t�rm term term", new MockAnalyzer(MockTokenizer.WHITESPACE, false),
+    assertQueryEquals("t�rm term term", new MockAnalyzer(random, MockTokenizer.WHITESPACE, false),
         "t�rm term term");
-    assertQueryEquals("�mlaut", new MockAnalyzer(MockTokenizer.WHITESPACE, false), "�mlaut");
+    assertQueryEquals("�mlaut", new MockAnalyzer(random, MockTokenizer.WHITESPACE, false), "�mlaut");
 
     // FIXME: change MockAnalyzer to not extend CharTokenizer for this test
     //assertQueryEquals("\"\"", new KeywordAnalyzer(), "");
@@ -470,7 +470,7 @@ public class TestQPHelper extends Lucene
   }
 
   public void testPunct() throws Exception {
-    Analyzer a = new MockAnalyzer(MockTokenizer.WHITESPACE, false);
+    Analyzer a = new MockAnalyzer(random, MockTokenizer.WHITESPACE, false);
     assertQueryEquals("a&b", a, "a&b");
     assertQueryEquals("a&&b", a, "a&&b");
     assertQueryEquals(".NET", a, ".NET");
@@ -491,7 +491,7 @@ public class TestQPHelper extends Lucene
     assertQueryEquals("term 1.0 1 2", null, "term");
     assertQueryEquals("term term1 term2", null, "term term term");
 
-    Analyzer a = new MockAnalyzer(MockTokenizer.WHITESPACE, false);
+    Analyzer a = new MockAnalyzer(random, MockTokenizer.WHITESPACE, false);
     assertQueryEquals("3", a, "3");
     assertQueryEquals("term 1.0 1 2", a, "term 1.0 1 2");
     assertQueryEquals("term term1 term2", a, "term term1 term2");
@@ -726,7 +726,7 @@ public class TestQPHelper extends Lucene
   }
 
   public void testEscaped() throws Exception {
-    Analyzer a = new MockAnalyzer(MockTokenizer.WHITESPACE, false);
+    Analyzer a = new MockAnalyzer(random, MockTokenizer.WHITESPACE, false);
 
     /*
      * assertQueryEquals("\\[brackets", a, "\\[brackets");
@@ -825,7 +825,7 @@ public class TestQPHelper extends Lucene
   }
 
   public void testQueryStringEscaping() throws Exception {
-    Analyzer a = new MockAnalyzer(MockTokenizer.WHITESPACE, false);
+    Analyzer a = new MockAnalyzer(random, MockTokenizer.WHITESPACE, false);
 
     assertEscapedQueryEquals("a-b:c", a, "a\\-b\\:c");
     assertEscapedQueryEquals("a+b:c", a, "a\\+b\\:c");
@@ -866,7 +866,7 @@ public class TestQPHelper extends Lucene
   @Ignore("contrib queryparser shouldn't escape wildcard terms")
   public void testEscapedWildcard() throws Exception {
     StandardQueryParser qp = new StandardQueryParser();
-    qp.setAnalyzer(new MockAnalyzer(MockTokenizer.WHITESPACE, false));
+    qp.setAnalyzer(new MockAnalyzer(random, MockTokenizer.WHITESPACE, false));
 
     WildcardQuery q = new WildcardQuery(new Term("field", "foo\\?ba?r"));
     assertEquals(q, qp.parse("foo\\?ba?r", "field"));
@@ -904,7 +904,7 @@ public class TestQPHelper extends Lucene
 
   public void testBoost() throws Exception {
     CharacterRunAutomaton stopSet = new CharacterRunAutomaton(BasicAutomata.makeString("on"));
-    Analyzer oneStopAnalyzer = new MockAnalyzer(MockTokenizer.SIMPLE, true, stopSet, true);
+    Analyzer oneStopAnalyzer = new MockAnalyzer(random, MockTokenizer.SIMPLE, true, stopSet, true);
     StandardQueryParser qp = new StandardQueryParser();
     qp.setAnalyzer(oneStopAnalyzer);
 
@@ -920,7 +920,7 @@ public class TestQPHelper extends Lucene
     assertNotNull(q);
 
     StandardQueryParser qp2 = new StandardQueryParser();
-    qp2.setAnalyzer(new MockAnalyzer(MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET, true));
+    qp2.setAnalyzer(new MockAnalyzer(random, MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET, true));
 
     q = qp2.parse("the^3", "field");
     // "the" is a stop word so the result is an empty query:
@@ -950,7 +950,7 @@ public class TestQPHelper extends Lucene
 
   public void testCustomQueryParserWildcard() {
     try {
-      new QPTestParser(new MockAnalyzer(MockTokenizer.WHITESPACE, false)).parse("a?t", "contents");
+      new QPTestParser(new MockAnalyzer(random, MockTokenizer.WHITESPACE, false)).parse("a?t", "contents");
       fail("Wildcard queries should not be allowed");
     } catch (QueryNodeException expected) {
       // expected exception
@@ -959,7 +959,7 @@ public class TestQPHelper extends Lucene
 
   public void testCustomQueryParserFuzzy() throws Exception {
     try {
-      new QPTestParser(new MockAnalyzer(MockTokenizer.WHITESPACE, false)).parse("xunit~", "contents");
+      new QPTestParser(new MockAnalyzer(random, MockTokenizer.WHITESPACE, false)).parse("xunit~", "contents");
       fail("Fuzzy queries should not be allowed");
     } catch (QueryNodeException expected) {
       // expected exception
@@ -970,7 +970,7 @@ public class TestQPHelper extends Lucene
     BooleanQuery.setMaxClauseCount(2);
     try {
       StandardQueryParser qp = new StandardQueryParser();
-      qp.setAnalyzer(new MockAnalyzer(MockTokenizer.WHITESPACE, false));
+      qp.setAnalyzer(new MockAnalyzer(random, MockTokenizer.WHITESPACE, false));
 
       qp.parse("one two three", "field");
       fail("ParseException expected due to too many boolean clauses");
@@ -984,7 +984,7 @@ public class TestQPHelper extends Lucene
    */
   public void testPrecedence() throws Exception {
     StandardQueryParser qp = new StandardQueryParser();
-    qp.setAnalyzer(new MockAnalyzer(MockTokenizer.WHITESPACE, false));
+    qp.setAnalyzer(new MockAnalyzer(random, MockTokenizer.WHITESPACE, false));
 
     Query query1 = qp.parse("A AND B OR C AND D", "field");
     Query query2 = qp.parse("+A +B +C +D", "field");
@@ -995,7 +995,7 @@ public class TestQPHelper extends Lucene
 // Todo: Convert from DateField to DateUtil
 //  public void testLocalDateFormat() throws IOException, QueryNodeException {
 //    Directory ramDir = newDirectory();
-//    IndexWriter iw = new IndexWriter(ramDir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(MockTokenizer.WHITESPACE, false)));
+//    IndexWriter iw = new IndexWriter(ramDir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random, MockTokenizer.WHITESPACE, false)));
 //    addDateDoc("a", 2005, 12, 2, 10, 15, 33, iw);
 //    addDateDoc("b", 2005, 12, 4, 22, 15, 00, iw);
 //    iw.close();
@@ -1116,7 +1116,7 @@ public class TestQPHelper extends Lucene
   public void testStopwords() throws Exception {
     StandardQueryParser qp = new StandardQueryParser();
     CharacterRunAutomaton stopSet = new CharacterRunAutomaton(new RegExp("the|foo").toAutomaton());
-    qp.setAnalyzer(new MockAnalyzer(MockTokenizer.SIMPLE, true, stopSet, true));
+    qp.setAnalyzer(new MockAnalyzer(random, MockTokenizer.SIMPLE, true, stopSet, true));
 
     Query result = qp.parse("a:the OR a:foo", "a");
     assertNotNull("result is null and it shouldn't be", result);
@@ -1140,7 +1140,7 @@ public class TestQPHelper extends Lucene
   public void testPositionIncrement() throws Exception {
     StandardQueryParser qp = new StandardQueryParser();
     qp.setAnalyzer(
-        new MockAnalyzer(MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET, true));
+        new MockAnalyzer(random, MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET, true));
 
     qp.setEnablePositionIncrements(true);
 
@@ -1161,7 +1161,7 @@ public class TestQPHelper extends Lucene
 
   public void testMatchAllDocs() throws Exception {
     StandardQueryParser qp = new StandardQueryParser();
-    qp.setAnalyzer(new MockAnalyzer(MockTokenizer.WHITESPACE, false));
+    qp.setAnalyzer(new MockAnalyzer(random, MockTokenizer.WHITESPACE, false));
 
     assertEquals(new MatchAllDocsQuery(), qp.parse("*:*", "field"));
     assertEquals(new MatchAllDocsQuery(), qp.parse("(*:*)", "field"));
@@ -1173,7 +1173,7 @@ public class TestQPHelper extends Lucene
   private void assertHits(int expected, String query, IndexSearcher is)
       throws IOException, QueryNodeException {
     StandardQueryParser qp = new StandardQueryParser();
-    qp.setAnalyzer(new MockAnalyzer(MockTokenizer.WHITESPACE, false));
+    qp.setAnalyzer(new MockAnalyzer(random, MockTokenizer.WHITESPACE, false));
     qp.setLocale(Locale.ENGLISH);
 
     Query q = qp.parse(query, "date");

Modified: lucene/dev/branches/flexscoring/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/surround/query/SingleFieldTestDb.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/surround/query/SingleFieldTestDb.java?rev=1103112&r1=1103111&r2=1103112&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/surround/query/SingleFieldTestDb.java (original)
+++ lucene/dev/branches/flexscoring/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/surround/query/SingleFieldTestDb.java Sat May 14 13:51:35 2011
@@ -41,7 +41,7 @@ public class SingleFieldTestDb {
       fieldName = fName;
       IndexWriter writer = new IndexWriter(db, new IndexWriterConfig(
           Version.LUCENE_CURRENT,
-          new MockAnalyzer()));
+          new MockAnalyzer(random)));
       for (int j = 0; j < docs.length; j++) {
         Document d = new Document();
         d.add(new Field(fieldName, docs[j], Field.Store.NO, Field.Index.ANALYZED));

Modified: lucene/dev/branches/flexscoring/lucene/contrib/spatial/src/test/org/apache/lucene/spatial/tier/TestCartesian.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/contrib/spatial/src/test/org/apache/lucene/spatial/tier/TestCartesian.java?rev=1103112&r1=1103111&r2=1103112&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/contrib/spatial/src/test/org/apache/lucene/spatial/tier/TestCartesian.java (original)
+++ lucene/dev/branches/flexscoring/lucene/contrib/spatial/src/test/org/apache/lucene/spatial/tier/TestCartesian.java Sat May 14 13:51:35 2011
@@ -71,7 +71,7 @@ public class TestCartesian extends Lucen
     super.setUp();
     directory = newDirectory();
 
-    IndexWriter writer = new IndexWriter(directory, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()));
+    IndexWriter writer = new IndexWriter(directory, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)));
     
     setUpPlotter( 2, 15);
     

Modified: lucene/dev/branches/flexscoring/lucene/contrib/spatial/src/test/org/apache/lucene/spatial/tier/TestDistance.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/contrib/spatial/src/test/org/apache/lucene/spatial/tier/TestDistance.java?rev=1103112&r1=1103111&r2=1103112&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/contrib/spatial/src/test/org/apache/lucene/spatial/tier/TestDistance.java (original)
+++ lucene/dev/branches/flexscoring/lucene/contrib/spatial/src/test/org/apache/lucene/spatial/tier/TestDistance.java Sat May 14 13:51:35 2011
@@ -47,7 +47,7 @@ public class TestDistance extends Lucene
   public void setUp() throws Exception {
     super.setUp();
     directory = newDirectory();
-    writer = new IndexWriter(directory, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()));
+    writer = new IndexWriter(directory, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)));
     addData(writer);
     
   }

Modified: lucene/dev/branches/flexscoring/lucene/contrib/spellchecker/src/java/org/apache/lucene/search/spell/SpellChecker.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/contrib/spellchecker/src/java/org/apache/lucene/search/spell/SpellChecker.java?rev=1103112&r1=1103111&r2=1103112&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/contrib/spellchecker/src/java/org/apache/lucene/search/spell/SpellChecker.java (original)
+++ lucene/dev/branches/flexscoring/lucene/contrib/spellchecker/src/java/org/apache/lucene/search/spell/SpellChecker.java Sat May 14 13:51:35 2011
@@ -29,7 +29,7 @@ import org.apache.lucene.document.Field;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.index.LogMergePolicy;
+import org.apache.lucene.index.TieredMergePolicy;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.index.IndexWriterConfig.OpenMode;
 import org.apache.lucene.index.Terms;
@@ -45,7 +45,6 @@ import org.apache.lucene.store.Directory
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.ReaderUtil;
 import org.apache.lucene.util.Version;
-import org.apache.lucene.util.VirtualMethod;
 
 /**
  * <p>
@@ -508,7 +507,7 @@ public class SpellChecker implements jav
       ensureOpen();
       final Directory dir = this.spellIndex;
       final IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(Version.LUCENE_CURRENT, new WhitespaceAnalyzer(Version.LUCENE_CURRENT)).setRAMBufferSizeMB(ramMB));
-      ((LogMergePolicy) writer.getConfig().getMergePolicy()).setMergeFactor(mergeFactor);
+      ((TieredMergePolicy) writer.getConfig().getMergePolicy()).setMaxMergeAtOnce(mergeFactor);
       IndexSearcher indexSearcher = obtainSearcher();
       final List<TermsEnum> termsEnums = new ArrayList<TermsEnum>();
 

Modified: lucene/dev/branches/flexscoring/lucene/contrib/spellchecker/src/test/org/apache/lucene/search/spell/TestDirectSpellChecker.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/contrib/spellchecker/src/test/org/apache/lucene/search/spell/TestDirectSpellChecker.java?rev=1103112&r1=1103111&r2=1103112&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/contrib/spellchecker/src/test/org/apache/lucene/search/spell/TestDirectSpellChecker.java (original)
+++ lucene/dev/branches/flexscoring/lucene/contrib/spellchecker/src/test/org/apache/lucene/search/spell/TestDirectSpellChecker.java Sat May 14 13:51:35 2011
@@ -35,7 +35,7 @@ public class TestDirectSpellChecker exte
     spellChecker.setMinQueryLength(0);
     Directory dir = newDirectory();
     RandomIndexWriter writer = new RandomIndexWriter(random, dir, 
-        new MockAnalyzer(MockTokenizer.SIMPLE, true));
+        new MockAnalyzer(random, MockTokenizer.SIMPLE, true));
 
     for (int i = 0; i < 20; i++) {
       Document doc = new Document();
@@ -93,7 +93,7 @@ public class TestDirectSpellChecker exte
   public void testOptions() throws Exception {
     Directory dir = newDirectory();
     RandomIndexWriter writer = new RandomIndexWriter(random, dir, 
-        new MockAnalyzer(MockTokenizer.SIMPLE, true));
+        new MockAnalyzer(random, MockTokenizer.SIMPLE, true));
 
     Document doc = new Document();
     doc.add(newField("text", "foobar", Field.Store.NO, Field.Index.ANALYZED));

Modified: lucene/dev/branches/flexscoring/lucene/contrib/spellchecker/src/test/org/apache/lucene/search/spell/TestLuceneDictionary.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/contrib/spellchecker/src/test/org/apache/lucene/search/spell/TestLuceneDictionary.java?rev=1103112&r1=1103111&r2=1103112&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/contrib/spellchecker/src/test/org/apache/lucene/search/spell/TestLuceneDictionary.java (original)
+++ lucene/dev/branches/flexscoring/lucene/contrib/spellchecker/src/test/org/apache/lucene/search/spell/TestLuceneDictionary.java Sat May 14 13:51:35 2011
@@ -46,7 +46,7 @@ public class TestLuceneDictionary extend
   public void setUp() throws Exception {
     super.setUp();
     store = newDirectory();
-    IndexWriter writer = new IndexWriter(store, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(MockTokenizer.WHITESPACE, false)));
+    IndexWriter writer = new IndexWriter(store, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random, MockTokenizer.WHITESPACE, false)));
 
     Document doc;
 

Modified: lucene/dev/branches/flexscoring/lucene/contrib/spellchecker/src/test/org/apache/lucene/search/spell/TestSpellChecker.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/contrib/spellchecker/src/test/org/apache/lucene/search/spell/TestSpellChecker.java?rev=1103112&r1=1103111&r2=1103112&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/contrib/spellchecker/src/test/org/apache/lucene/search/spell/TestSpellChecker.java (original)
+++ lucene/dev/branches/flexscoring/lucene/contrib/spellchecker/src/test/org/apache/lucene/search/spell/TestSpellChecker.java Sat May 14 13:51:35 2011
@@ -54,7 +54,7 @@ public class TestSpellChecker extends Lu
     //create a user index
     userindex = newDirectory();
     IndexWriter writer = new IndexWriter(userindex, new IndexWriterConfig(
-        TEST_VERSION_CURRENT, new MockAnalyzer()));
+        TEST_VERSION_CURRENT, new MockAnalyzer(random)));
 
     for (int i = 0; i < 1000; i++) {
       Document doc = new Document();

Modified: lucene/dev/branches/flexscoring/lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/SynExpand.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/SynExpand.java?rev=1103112&r1=1103111&r2=1103112&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/SynExpand.java (original)
+++ lucene/dev/branches/flexscoring/lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/SynExpand.java Sat May 14 13:51:35 2011
@@ -116,14 +116,16 @@ public final class SynExpand {
 		if ( a == null) a = new StandardAnalyzer(Version.LUCENE_CURRENT);
 
 		// [1] Parse query into separate words so that when we expand we can avoid dups
-		TokenStream ts = a.tokenStream( field, new StringReader( query));
+		TokenStream ts = a.reusableTokenStream( field, new StringReader( query));
 		CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
-		
+		ts.reset();
 		while (ts.incrementToken()) {
 		  String word = termAtt.toString();
 			if ( already.add( word))
 				top.add( word);
 		}
+		ts.end();
+		ts.close();
 		final BooleanQuery tmp = new BooleanQuery();
 		
 		// [2] form query

Modified: lucene/dev/branches/flexscoring/lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/SynLookup.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/SynLookup.java?rev=1103112&r1=1103111&r2=1103112&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/SynLookup.java (original)
+++ lucene/dev/branches/flexscoring/lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/SynLookup.java Sat May 14 13:51:35 2011
@@ -124,7 +124,7 @@ public class SynLookup {
 		List<String> top = new LinkedList<String>(); // needs to be separately listed..
 
 		// [1] Parse query into separate words so that when we expand we can avoid dups
-		TokenStream ts = a.tokenStream( field, new StringReader( query));
+		TokenStream ts = a.reusableTokenStream( field, new StringReader( query));
     CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
     
 		while (ts.incrementToken()) {

Modified: lucene/dev/branches/flexscoring/lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/Syns2Index.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/Syns2Index.java?rev=1103112&r1=1103111&r2=1103112&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/Syns2Index.java (original)
+++ lucene/dev/branches/flexscoring/lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/Syns2Index.java Sat May 14 13:51:35 2011
@@ -36,7 +36,7 @@ import org.apache.lucene.document.Docume
 import org.apache.lucene.document.Field;
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.index.LogMergePolicy;
+import org.apache.lucene.index.TieredMergePolicy;
 import org.apache.lucene.index.IndexWriterConfig.OpenMode;
 import org.apache.lucene.store.FSDirectory;
 import org.apache.lucene.util.Version;
@@ -250,7 +250,7 @@ public class Syns2Index
           // override the specific index if it already exists
           IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(
               Version.LUCENE_CURRENT, ana).setOpenMode(OpenMode.CREATE));
-          ((LogMergePolicy) writer.getConfig().getMergePolicy()).setUseCompoundFile(true); // why?
+          ((TieredMergePolicy) writer.getConfig().getMergePolicy()).setUseCompoundFile(true); // why?
           Iterator<String> i1 = word2Nums.keySet().iterator();
           while (i1.hasNext()) // for each word
           {

Modified: lucene/dev/branches/flexscoring/lucene/contrib/wordnet/src/test/org/apache/lucene/wordnet/TestSynonymTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/contrib/wordnet/src/test/org/apache/lucene/wordnet/TestSynonymTokenFilter.java?rev=1103112&r1=1103111&r2=1103112&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/contrib/wordnet/src/test/org/apache/lucene/wordnet/TestSynonymTokenFilter.java (original)
+++ lucene/dev/branches/flexscoring/lucene/contrib/wordnet/src/test/org/apache/lucene/wordnet/TestSynonymTokenFilter.java Sat May 14 13:51:35 2011
@@ -111,7 +111,6 @@ public class TestSynonymTokenFilter exte
         setPreviousTokenStream(streams);
       } else {
         streams.source.reset(reader);
-        streams.result.reset(); // reset the SynonymTokenFilter
       }
       return streams.result;
     }

Modified: lucene/dev/branches/flexscoring/lucene/contrib/wordnet/src/test/org/apache/lucene/wordnet/TestWordnet.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/contrib/wordnet/src/test/org/apache/lucene/wordnet/TestWordnet.java?rev=1103112&r1=1103111&r2=1103112&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/contrib/wordnet/src/test/org/apache/lucene/wordnet/TestWordnet.java (original)
+++ lucene/dev/branches/flexscoring/lucene/contrib/wordnet/src/test/org/apache/lucene/wordnet/TestWordnet.java Sat May 14 13:51:35 2011
@@ -29,6 +29,7 @@ import org.apache.lucene.search.Query;
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util._TestUtil;
 
 public class TestWordnet extends LuceneTestCase {
   private IndexSearcher searcher;
@@ -42,6 +43,7 @@ public class TestWordnet extends LuceneT
     // create a temporary synonym index
     File testFile = getDataFile("testSynonyms.txt");
     String commandLineArgs[] = { testFile.getAbsolutePath(), storePathName };
+    _TestUtil.rmDir(new File(storePathName));
     
     try {
       Syns2Index.main(commandLineArgs);
@@ -61,7 +63,7 @@ public class TestWordnet extends LuceneT
   
   private void assertExpandsTo(String term, String expected[]) throws IOException {
     Query expandedQuery = SynExpand.expand(term, searcher, new 
-        MockAnalyzer(), "field", 1F);
+        MockAnalyzer(random), "field", 1F);
     BooleanQuery expectedQuery = new BooleanQuery();
     for (String t : expected)
       expectedQuery.add(new TermQuery(new Term("field", t)), 
@@ -71,8 +73,12 @@ public class TestWordnet extends LuceneT
 
   @Override
   public void tearDown() throws Exception {
-    searcher.close();
-    dir.close();
+    if (searcher != null) {
+      searcher.close();
+    }
+    if (dir != null) {
+      dir.close();
+    }
     rmDir(storePathName); // delete our temporary synonym index
     super.tearDown();
   }

Modified: lucene/dev/branches/flexscoring/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/LikeThisQueryBuilder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/LikeThisQueryBuilder.java?rev=1103112&r1=1103111&r2=1103112&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/LikeThisQueryBuilder.java (original)
+++ lucene/dev/branches/flexscoring/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/LikeThisQueryBuilder.java Sat May 14 13:51:35 2011
@@ -76,13 +76,16 @@ public class LikeThisQueryBuilder implem
 		    stopWordsSet=new HashSet<String>();
 		    for (int i = 0; i < fields.length; i++)
             {
-                TokenStream ts = analyzer.tokenStream(fields[i],new StringReader(stopWords));
-                CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
                 try
                 {
+                  TokenStream ts = analyzer.reusableTokenStream(fields[i],new StringReader(stopWords));
+                  CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
+                  ts.reset();
 	                while(ts.incrementToken()) {
 	                    stopWordsSet.add(termAtt.toString());
 	                }
+	                ts.end();
+	                ts.close();
                 }
                 catch(IOException ioe)
                 {

Modified: lucene/dev/branches/flexscoring/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/SpanOrTermsBuilder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/SpanOrTermsBuilder.java?rev=1103112&r1=1103111&r2=1103112&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/SpanOrTermsBuilder.java (original)
+++ lucene/dev/branches/flexscoring/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/SpanOrTermsBuilder.java Sat May 14 13:51:35 2011
@@ -56,14 +56,17 @@ public class SpanOrTermsBuilder extends 
 		try
 		{
 			ArrayList<SpanQuery> clausesList=new ArrayList<SpanQuery>();
-			TokenStream ts=analyzer.tokenStream(fieldName,new StringReader(value));
+			TokenStream ts=analyzer.reusableTokenStream(fieldName,new StringReader(value));
 			TermToBytesRefAttribute termAtt = ts.addAttribute(TermToBytesRefAttribute.class);
       BytesRef bytes = termAtt.getBytesRef();
+      ts.reset();
 	    while (ts.incrementToken()) {
 	        termAtt.fillBytesRef();
 			    SpanTermQuery stq=new SpanTermQuery(new Term(fieldName, new BytesRef(bytes)));
 			    clausesList.add(stq);
 			}
+	    ts.end();
+	    ts.close();
 			SpanOrQuery soq=new SpanOrQuery(clausesList.toArray(new SpanQuery[clausesList.size()]));
 			soq.setBoost(DOMUtils.getAttribute(e,"boost",1.0f));
 			return soq;

Modified: lucene/dev/branches/flexscoring/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsFilterBuilder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsFilterBuilder.java?rev=1103112&r1=1103111&r2=1103112&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsFilterBuilder.java (original)
+++ lucene/dev/branches/flexscoring/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsFilterBuilder.java Sat May 14 13:51:35 2011
@@ -57,13 +57,14 @@ public class TermsFilterBuilder implemen
 		TermsFilter tf = new TermsFilter();
 		String text = DOMUtils.getNonBlankTextOrFail(e);
 		String fieldName = DOMUtils.getAttributeWithInheritanceOrFail(e, "fieldName");
-		TokenStream ts = analyzer.tokenStream(fieldName, new StringReader(text));
-    TermToBytesRefAttribute termAtt = ts.addAttribute(TermToBytesRefAttribute.class);
     
 		try
 		{
+	    TokenStream ts = analyzer.reusableTokenStream(fieldName, new StringReader(text));
+	    TermToBytesRefAttribute termAtt = ts.addAttribute(TermToBytesRefAttribute.class);
 			Term term = null;
       BytesRef bytes = termAtt.getBytesRef();
+      ts.reset();
 	      while (ts.incrementToken()) {
 	        termAtt.fillBytesRef();
 				if (term == null)
@@ -76,6 +77,8 @@ public class TermsFilterBuilder implemen
 				}
 				tf.addTerm(term);
 			}
+	    ts.end();
+	    ts.close();
 		} 
 		catch (IOException ioe)
 		{

Modified: lucene/dev/branches/flexscoring/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsQueryBuilder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsQueryBuilder.java?rev=1103112&r1=1103111&r2=1103112&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsQueryBuilder.java (original)
+++ lucene/dev/branches/flexscoring/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsQueryBuilder.java Sat May 14 13:51:35 2011
@@ -55,12 +55,13 @@ public class TermsQueryBuilder implement
  		
 		BooleanQuery bq=new BooleanQuery(DOMUtils.getAttribute(e,"disableCoord",false));
 		bq.setMinimumNumberShouldMatch(DOMUtils.getAttribute(e,"minimumNumberShouldMatch",0));
-		TokenStream ts = analyzer.tokenStream(fieldName, new StringReader(text));
 		try
 		{
+	    TokenStream ts = analyzer.reusableTokenStream(fieldName, new StringReader(text));
 		  TermToBytesRefAttribute termAtt = ts.addAttribute(TermToBytesRefAttribute.class);
 			Term term = null;
       BytesRef bytes = termAtt.getBytesRef();
+      ts.reset();
 			while (ts.incrementToken()) {
         termAtt.fillBytesRef();
 				if (term == null)
@@ -73,6 +74,8 @@ public class TermsQueryBuilder implement
 				}
 				bq.add(new BooleanClause(new TermQuery(term),BooleanClause.Occur.SHOULD));
 			}
+			ts.end();
+			ts.close();
 		} 
 		catch (IOException ioe)
 		{

Modified: lucene/dev/branches/flexscoring/lucene/contrib/xml-query-parser/src/test/org/apache/lucene/xmlparser/TestParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/contrib/xml-query-parser/src/test/org/apache/lucene/xmlparser/TestParser.java?rev=1103112&r1=1103111&r2=1103112&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/contrib/xml-query-parser/src/test/org/apache/lucene/xmlparser/TestParser.java (original)
+++ lucene/dev/branches/flexscoring/lucene/contrib/xml-query-parser/src/test/org/apache/lucene/xmlparser/TestParser.java Sat May 14 13:51:35 2011
@@ -49,7 +49,7 @@ public class TestParser extends LuceneTe
 	@BeforeClass
 	public static void beforeClass() throws Exception {
 	  // TODO: rewrite test (this needs to set QueryParser.enablePositionIncrements, too, for work with CURRENT):
-	  Analyzer analyzer=new MockAnalyzer(MockTokenizer.WHITESPACE, true, MockTokenFilter.ENGLISH_STOPSET, false); 
+	  Analyzer analyzer=new MockAnalyzer(random, MockTokenizer.WHITESPACE, true, MockTokenFilter.ENGLISH_STOPSET, false); 
     //initialize the parser
 	  builder=new CorePlusExtensionsParser("contents",analyzer);
 		
@@ -187,7 +187,8 @@ public class TestParser extends LuceneTe
 	}
 	public void testDuplicateFilterQueryXML() throws ParserException, IOException
 	{
-                        Assume.assumeTrue(searcher.getIndexReader().getSequentialSubReaders().length == 1);
+      Assume.assumeTrue(searcher.getIndexReader().getSequentialSubReaders() == null || 
+                        searcher.getIndexReader().getSequentialSubReaders().length == 1);
 			Query q=parse("DuplicateFilterQuery.xml");
 			int h = searcher.search(q, null, 1000).totalHits;
 			assertEquals("DuplicateFilterQuery should produce 1 result ", 1,h);

Modified: lucene/dev/branches/flexscoring/lucene/contrib/xml-query-parser/src/test/org/apache/lucene/xmlparser/TestQueryTemplateManager.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/contrib/xml-query-parser/src/test/org/apache/lucene/xmlparser/TestQueryTemplateManager.java?rev=1103112&r1=1103111&r2=1103112&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/contrib/xml-query-parser/src/test/org/apache/lucene/xmlparser/TestQueryTemplateManager.java (original)
+++ lucene/dev/branches/flexscoring/lucene/contrib/xml-query-parser/src/test/org/apache/lucene/xmlparser/TestQueryTemplateManager.java Sat May 14 13:51:35 2011
@@ -44,7 +44,7 @@ import org.xml.sax.SAXException;
 public class TestQueryTemplateManager extends LuceneTestCase {
 
 	CoreParser builder;
-	Analyzer analyzer=new MockAnalyzer();
+	Analyzer analyzer=new MockAnalyzer(random);
 	private IndexSearcher searcher;
 	private Directory dir;
 	

Modified: lucene/dev/branches/flexscoring/lucene/docs/contributions.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/docs/contributions.html?rev=1103112&r1=1103111&r2=1103112&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/docs/contributions.html (original)
+++ lucene/dev/branches/flexscoring/lucene/docs/contributions.html Sat May 14 13:51:35 2011
@@ -3,7 +3,7 @@
 <head>
 <META http-equiv="Content-Type" content="text/html; charset=UTF-8">
 <meta content="Apache Forrest" name="Generator">
-<meta name="Forrest-version" content="0.8">
+<meta name="Forrest-version" content="0.9">
 <meta name="Forrest-skin-name" content="lucene">
 <title>
 	Apache Lucene - Contributions
@@ -275,7 +275,7 @@ document.write("Last Published: " + docu
 <a href="#PDFTextStream -- PDF text and metadata extraction">PDFTextStream -- PDF text and metadata extraction</a>
 </li>
 <li>
-<a href="#PJ Classic & PJ Professional - PDF Document Conversion">PJ Classic &amp; PJ Professional - PDF Document Conversion</a>
+<a href="#PJ Classic &amp; PJ Professional - PDF Document Conversion">PJ Classic &amp; PJ Professional - PDF Document Conversion</a>
 </li>
 </ul>
 </li>
@@ -403,7 +403,7 @@ document.write("Last Published: " + docu
                             URL
                         </th>
                         <td>
-                            <a href="http://marc.theaimsgroup.com/?l=lucene-dev&m=100723333506246&w=2">
+                            <a href="http://marc.theaimsgroup.com/?l=lucene-dev&amp;m=100723333506246&amp;w=2">
                                 http://marc.theaimsgroup.com/?l=lucene-dev&amp;m=100723333506246&amp;w=2
                             </a>
                         </td>
@@ -538,7 +538,7 @@ document.write("Last Published: " + docu
 </tr>
                 
 </table>
-<a name="N10124"></a><a name="PJ Classic & PJ Professional - PDF Document Conversion"></a>
+<a name="N10124"></a><a name="PJ Classic &amp; PJ Professional - PDF Document Conversion"></a>
 <h3 class="boxed">PJ Classic &amp; PJ Professional - PDF Document Conversion</h3>
 <table class="ForrestTable" cellspacing="1" cellpadding="4">
                     

Modified: lucene/dev/branches/flexscoring/lucene/docs/contributions.pdf
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/docs/contributions.pdf?rev=1103112&r1=1103111&r2=1103112&view=diff
==============================================================================
Files lucene/dev/branches/flexscoring/lucene/docs/contributions.pdf (original) and lucene/dev/branches/flexscoring/lucene/docs/contributions.pdf Sat May 14 13:51:35 2011 differ

Modified: lucene/dev/branches/flexscoring/lucene/docs/demo.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/docs/demo.html?rev=1103112&r1=1103111&r2=1103112&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/docs/demo.html (original)
+++ lucene/dev/branches/flexscoring/lucene/docs/demo.html Sat May 14 13:51:35 2011
@@ -3,7 +3,7 @@
 <head>
 <META http-equiv="Content-Type" content="text/html; charset=UTF-8">
 <meta content="Apache Forrest" name="Generator">
-<meta name="Forrest-version" content="0.8">
+<meta name="Forrest-version" content="0.9">
 <meta name="Forrest-skin-name" content="lucene">
 <title>
 	Apache Lucene - Building and Installing the Basic Demo

Modified: lucene/dev/branches/flexscoring/lucene/docs/demo.pdf
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/docs/demo.pdf?rev=1103112&r1=1103111&r2=1103112&view=diff
==============================================================================
Files lucene/dev/branches/flexscoring/lucene/docs/demo.pdf (original) and lucene/dev/branches/flexscoring/lucene/docs/demo.pdf Sat May 14 13:51:35 2011 differ

Modified: lucene/dev/branches/flexscoring/lucene/docs/demo2.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/docs/demo2.html?rev=1103112&r1=1103111&r2=1103112&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/docs/demo2.html (original)
+++ lucene/dev/branches/flexscoring/lucene/docs/demo2.html Sat May 14 13:51:35 2011
@@ -3,7 +3,7 @@
 <head>
 <META http-equiv="Content-Type" content="text/html; charset=UTF-8">
 <meta content="Apache Forrest" name="Generator">
-<meta name="Forrest-version" content="0.8">
+<meta name="Forrest-version" content="0.9">
 <meta name="Forrest-skin-name" content="lucene">
 <title>
 	Apache Lucene - Basic Demo Sources Walk-through

Modified: lucene/dev/branches/flexscoring/lucene/docs/demo2.pdf
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/docs/demo2.pdf?rev=1103112&r1=1103111&r2=1103112&view=diff
==============================================================================
Files lucene/dev/branches/flexscoring/lucene/docs/demo2.pdf (original) and lucene/dev/branches/flexscoring/lucene/docs/demo2.pdf Sat May 14 13:51:35 2011 differ

Modified: lucene/dev/branches/flexscoring/lucene/docs/fileformats.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/docs/fileformats.html?rev=1103112&r1=1103111&r2=1103112&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/docs/fileformats.html (original)
+++ lucene/dev/branches/flexscoring/lucene/docs/fileformats.html Sat May 14 13:51:35 2011
@@ -3,7 +3,7 @@
 <head>
 <META http-equiv="Content-Type" content="text/html; charset=UTF-8">
 <meta content="Apache Forrest" name="Generator">
-<meta name="Forrest-version" content="0.8">
+<meta name="Forrest-version" content="0.9">
 <meta name="Forrest-skin-name" content="lucene">
 <title>
             Apache Lucene - Index File Formats
@@ -425,11 +425,19 @@ document.write("Last Published: " + docu
 <p>
             In version 3.1, segments records the code version
             that created them. See LUCENE-2720 for details.
+            
+            Additionally segments track explicitly whether or
+            not they have term vectors. See LUCENE-2811 for details.
+           </p>
+<p>
+            In version 3.2, numeric fields are written as natively
+            to stored fields file, previously they were stored in
+            text format only.
            </p>
 </div>
 
         
-<a name="N10037"></a><a name="Definitions"></a>
+<a name="N1003A"></a><a name="Definitions"></a>
 <h2 class="boxed">Definitions</h2>
 <div class="section">
 <p>
@@ -470,7 +478,7 @@ document.write("Last Published: " + docu
                 strings, the first naming the field, and the second naming text
                 within the field.
             </p>
-<a name="N10057"></a><a name="Inverted Indexing"></a>
+<a name="N1005A"></a><a name="Inverted Indexing"></a>
 <h3 class="boxed">Inverted Indexing</h3>
 <p>
                     The index stores statistics about terms in order
@@ -480,7 +488,7 @@ document.write("Last Published: " + docu
                     it.  This is the inverse of the natural relationship, in which
                     documents list terms.
                 </p>
-<a name="N10063"></a><a name="Types of Fields"></a>
+<a name="N10066"></a><a name="Types of Fields"></a>
 <h3 class="boxed">Types of Fields</h3>
 <p>
                     In Lucene, fields may be <i>stored</i>, in which
@@ -494,7 +502,7 @@ document.write("Last Published: " + docu
                     to be indexed literally.
                 </p>
 <p>See the <a href="api/core/org/apache/lucene/document/Field.html">Field</a> java docs for more information on Fields.</p>
-<a name="N10080"></a><a name="Segments"></a>
+<a name="N10083"></a><a name="Segments"></a>
 <h3 class="boxed">Segments</h3>
 <p>
                     Lucene indexes may be composed of multiple sub-indexes, or
@@ -520,7 +528,7 @@ document.write("Last Published: " + docu
                     Searches may involve multiple segments and/or multiple indexes, each
                     index potentially composed of a set of segments.
                 </p>
-<a name="N1009E"></a><a name="Document Numbers"></a>
+<a name="N100A1"></a><a name="Document Numbers"></a>
 <h3 class="boxed">Document Numbers</h3>
 <p>
                     Internally, Lucene refers to documents by an integer <i>document
@@ -575,7 +583,7 @@ document.write("Last Published: " + docu
 </div>
 
         
-<a name="N100C5"></a><a name="Overview"></a>
+<a name="N100C8"></a><a name="Overview"></a>
 <h2 class="boxed">Overview</h2>
 <div class="section">
 <p>
@@ -674,7 +682,7 @@ document.write("Last Published: " + docu
 </div>
 
         
-<a name="N10108"></a><a name="File Naming"></a>
+<a name="N1010B"></a><a name="File Naming"></a>
 <h2 class="boxed">File Naming</h2>
 <div class="section">
 <p>
@@ -701,7 +709,7 @@ document.write("Last Published: " + docu
             </p>
 </div>
       
-<a name="N10117"></a><a name="file-names"></a>
+<a name="N1011A"></a><a name="file-names"></a>
 <h2 class="boxed">Summary of File Extensions</h2>
 <div class="section">
 <p>The following table summarizes the names and extensions of the files in Lucene:
@@ -843,10 +851,10 @@ document.write("Last Published: " + docu
 </div>
 
         
-<a name="N10201"></a><a name="Primitive Types"></a>
+<a name="N10204"></a><a name="Primitive Types"></a>
 <h2 class="boxed">Primitive Types</h2>
 <div class="section">
-<a name="N10206"></a><a name="Byte"></a>
+<a name="N10209"></a><a name="Byte"></a>
 <h3 class="boxed">Byte</h3>
 <p>
                     The most primitive type
@@ -854,7 +862,7 @@ document.write("Last Published: " + docu
                     other data types are defined as sequences
                     of bytes, so file formats are byte-order independent.
                 </p>
-<a name="N1020F"></a><a name="UInt32"></a>
+<a name="N10212"></a><a name="UInt32"></a>
 <h3 class="boxed">UInt32</h3>
 <p>
                     32-bit unsigned integers are written as four
@@ -864,7 +872,7 @@ document.write("Last Published: " + docu
                     UInt32    --&gt; &lt;Byte&gt;<sup>4</sup>
                 
 </p>
-<a name="N1021E"></a><a name="Uint64"></a>
+<a name="N10221"></a><a name="Uint64"></a>
 <h3 class="boxed">Uint64</h3>
 <p>
                     64-bit unsigned integers are written as eight
@@ -873,7 +881,7 @@ document.write("Last Published: " + docu
 <p>UInt64    --&gt; &lt;Byte&gt;<sup>8</sup>
                 
 </p>
-<a name="N1022D"></a><a name="VInt"></a>
+<a name="N10230"></a><a name="VInt"></a>
 <h3 class="boxed">VInt</h3>
 <p>
                     A variable-length format for positive integers is
@@ -1423,13 +1431,13 @@ document.write("Last Published: " + docu
                     This provides compression while still being
                     efficient to decode.
                 </p>
-<a name="N10512"></a><a name="Chars"></a>
+<a name="N10515"></a><a name="Chars"></a>
 <h3 class="boxed">Chars</h3>
 <p>
                     Lucene writes unicode
                     character sequences as UTF-8 encoded bytes.
                 </p>
-<a name="N1051B"></a><a name="String"></a>
+<a name="N1051E"></a><a name="String"></a>
 <h3 class="boxed">String</h3>
 <p>
 		    Lucene writes strings as UTF-8 encoded bytes.
@@ -1442,10 +1450,10 @@ document.write("Last Published: " + docu
 </div>
 
         
-<a name="N10528"></a><a name="Compound Types"></a>
+<a name="N1052B"></a><a name="Compound Types"></a>
 <h2 class="boxed">Compound Types</h2>
 <div class="section">
-<a name="N1052D"></a><a name="MapStringString"></a>
+<a name="N10530"></a><a name="MapStringString"></a>
 <h3 class="boxed">Map&lt;String,String&gt;</h3>
 <p>
 		    In a couple places Lucene stores a Map
@@ -1458,13 +1466,13 @@ document.write("Last Published: " + docu
 </div>
 
         
-<a name="N1053D"></a><a name="Per-Index Files"></a>
+<a name="N10540"></a><a name="Per-Index Files"></a>
 <h2 class="boxed">Per-Index Files</h2>
 <div class="section">
 <p>
                 The files in this section exist one-per-index.
             </p>
-<a name="N10545"></a><a name="Segments File"></a>
+<a name="N10548"></a><a name="Segments File"></a>
 <h3 class="boxed">Segments File</h3>
 <p>
                     The active segments in the index are stored in the
@@ -1508,7 +1516,7 @@ document.write("Last Published: " + docu
 <b>3.1</b>
                     Segments --&gt; Format, Version, NameCounter, SegCount, &lt;SegVersion, SegName, SegSize, DelGen, DocStoreOffset, [DocStoreSegment, DocStoreIsCompoundFile], HasSingleNormFile, NumField,
                     NormGen<sup>NumField</sup>,
-                    IsCompoundFile, DeletionCount, HasProx, Diagnostics&gt;<sup>SegCount</sup>, CommitUserData, Checksum
+                    IsCompoundFile, DeletionCount, HasProx, Diagnostics, HasVectors&gt;<sup>SegCount</sup>, CommitUserData, Checksum
                 </p>
 <p>
                     Format, NameCounter, SegCount, SegSize, NumField,
@@ -1525,7 +1533,7 @@ document.write("Last Published: " + docu
 		</p>
 <p>
                     IsCompoundFile, HasSingleNormFile,
-                    DocStoreIsCompoundFile, HasProx --&gt; Int8
+                    DocStoreIsCompoundFile, HasProx, HasVectors --&gt; Int8
                 </p>
 <p>
 		    CommitUserData --&gt; Map&lt;String,String&gt;
@@ -1634,7 +1642,10 @@ document.write("Last Published: " + docu
 		    Lucene version, OS, Java version, why the segment
 		    was created (merge, flush, addIndexes), etc.
                 </p>
-<a name="N105CD"></a><a name="Lock File"></a>
+<p> HasVectors is 1 if this segment stores term vectors,
+            else it's 0.
+                </p>
+<a name="N105D3"></a><a name="Lock File"></a>
 <h3 class="boxed">Lock File</h3>
 <p>
                     The write lock, which is stored in the index
@@ -1648,14 +1659,14 @@ document.write("Last Published: " + docu
                     documents).  This lock file ensures that only one
                     writer is modifying the index at a time.
                 </p>
-<a name="N105D6"></a><a name="Deletable File"></a>
+<a name="N105DC"></a><a name="Deletable File"></a>
 <h3 class="boxed">Deletable File</h3>
 <p>
                     A writer dynamically computes
                     the files that are deletable, instead, so no file
                     is written.
                 </p>
-<a name="N105DF"></a><a name="Compound Files"></a>
+<a name="N105E5"></a><a name="Compound Files"></a>
 <h3 class="boxed">Compound Files</h3>
 <p>Starting with Lucene 1.4 the compound file format became default. This
                     is simply a container for all files described in the next section
@@ -1682,14 +1693,14 @@ document.write("Last Published: " + docu
 </div>
 
         
-<a name="N10607"></a><a name="Per-Segment Files"></a>
+<a name="N1060D"></a><a name="Per-Segment Files"></a>
 <h2 class="boxed">Per-Segment Files</h2>
 <div class="section">
 <p>
                 The remaining files are all per-segment, and are
                 thus defined by suffix.
             </p>
-<a name="N1060F"></a><a name="Fields"></a>
+<a name="N10615"></a><a name="Fields"></a>
 <h3 class="boxed">Fields</h3>
 <p>
                     
@@ -1862,13 +1873,29 @@ document.write("Last Published: " + docu
 <li>third bit is one for fields with compression option enabled
                                     (if compression is enabled, the algorithm used is ZLIB),
                                     only available for indexes until Lucene version 2.9.x</li>
+                                
+<li>4th to 6th bits (mask: 0x7&lt;&lt;3) define the type of a
+                                numeric field: <ul>
+                                  
+<li>all bits in mask are cleared if no numeric field at all</li>
+                                  
+<li>1&lt;&lt;3: Value is Int</li>
+                                  
+<li>2&lt;&lt;3: Value is Long</li>
+                                  
+<li>3&lt;&lt;3: Value is Int as Float (as of Integer.intBitsToFloat)</li>
+                                  
+<li>4&lt;&lt;3: Value is Long as Double (as of Double.longBitsToDouble)</li>
+                                
+</ul>
+</li>
                             
 </ul>
                         
 </p>
                         
 <p>Value --&gt;
-                            String | BinaryValue (depending on Bits)
+                            String | BinaryValue | Int | Long (depending on Bits)
                         </p>
                         
 <p>BinaryValue --&gt;
@@ -1883,7 +1910,7 @@ document.write("Last Published: " + docu
 </li>
                 
 </ol>
-<a name="N106B6"></a><a name="Term Dictionary"></a>
+<a name="N106D0"></a><a name="Term Dictionary"></a>
 <h3 class="boxed">Term Dictionary</h3>
 <p>
                     The term dictionary is represented as two files:
@@ -2075,7 +2102,7 @@ document.write("Last Published: " + docu
 </li>
                 
 </ol>
-<a name="N1073A"></a><a name="Frequencies"></a>
+<a name="N10754"></a><a name="Frequencies"></a>
 <h3 class="boxed">Frequencies</h3>
 <p>
                     The .frq file contains the lists of documents
@@ -2203,7 +2230,7 @@ document.write("Last Published: " + docu
                    entry in level-1. In the example has entry 15 on level 1 a pointer to entry 15 on level 0 and entry 31 on level 1 a pointer
                    to entry 31 on level 0.                   
                 </p>
-<a name="N107C2"></a><a name="Positions"></a>
+<a name="N107DC"></a><a name="Positions"></a>
 <h3 class="boxed">Positions</h3>
 <p>
                     The .prx file contains the lists of positions that
@@ -2273,7 +2300,7 @@ document.write("Last Published: " + docu
                     Payload. If PayloadLength is not stored, then this Payload has the same
                     length as the Payload at the previous position.
                 </p>
-<a name="N107FE"></a><a name="Normalization Factors"></a>
+<a name="N10818"></a><a name="Normalization Factors"></a>
 <h3 class="boxed">Normalization Factors</h3>
 <p>There's a single .nrm file containing all norms:
                 </p>
@@ -2353,7 +2380,7 @@ document.write("Last Published: " + docu
                 </p>
 <p>Separate norm files are created (when adequate) for both compound and non compound segments.
                 </p>
-<a name="N1084F"></a><a name="Term Vectors"></a>
+<a name="N10869"></a><a name="Term Vectors"></a>
 <h3 class="boxed">Term Vectors</h3>
 <p>
 		  Term Vector support is an optional on a field by
@@ -2489,7 +2516,7 @@ document.write("Last Published: " + docu
 </li>
                 
 </ol>
-<a name="N108EB"></a><a name="Deleted Documents"></a>
+<a name="N10905"></a><a name="Deleted Documents"></a>
 <h3 class="boxed">Deleted Documents</h3>
 <p>The .del file is
                     optional, and only exists when a segment contains deletions.
@@ -2553,7 +2580,7 @@ document.write("Last Published: " + docu
 </div>
 
         
-<a name="N10925"></a><a name="Limitations"></a>
+<a name="N1093F"></a><a name="Limitations"></a>
 <h2 class="boxed">Limitations</h2>
 <div class="section">
 <p>

Modified: lucene/dev/branches/flexscoring/lucene/docs/fileformats.pdf
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/docs/fileformats.pdf?rev=1103112&r1=1103111&r2=1103112&view=diff
==============================================================================
Files lucene/dev/branches/flexscoring/lucene/docs/fileformats.pdf (original) and lucene/dev/branches/flexscoring/lucene/docs/fileformats.pdf Sat May 14 13:51:35 2011 differ

Modified: lucene/dev/branches/flexscoring/lucene/docs/gettingstarted.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/docs/gettingstarted.html?rev=1103112&r1=1103111&r2=1103112&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/docs/gettingstarted.html (original)
+++ lucene/dev/branches/flexscoring/lucene/docs/gettingstarted.html Sat May 14 13:51:35 2011
@@ -3,7 +3,7 @@
 <head>
 <META http-equiv="Content-Type" content="text/html; charset=UTF-8">
 <meta content="Apache Forrest" name="Generator">
-<meta name="Forrest-version" content="0.8">
+<meta name="Forrest-version" content="0.9">
 <meta name="Forrest-skin-name" content="lucene">
 <title>
 	Apache Lucene - Getting Started Guide
@@ -268,15 +268,13 @@ may wish to skip sections.
 	
 <li>
 <a href="demo.html">About the command-line Lucene demo and its usage</a>.  This section
-	is intended for anyone who wants to use the command-line Lucene demo.</li> 
-<p></p>
+	is intended for anyone who wants to use the command-line Lucene demo.</li>
 
 	
 <li>
 <a href="demo2.html">About the sources and implementation for the command-line Lucene
 	demo</a>.  This section walks through the implementation details (sources) of the
-	command-line Lucene demo.  This section is intended for developers.</li> 
-<p></p>
+	command-line Lucene demo.  This section is intended for developers.</li>
 
 </ul>
 </div>

Modified: lucene/dev/branches/flexscoring/lucene/docs/gettingstarted.pdf
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/docs/gettingstarted.pdf?rev=1103112&r1=1103111&r2=1103112&view=diff
==============================================================================
Files lucene/dev/branches/flexscoring/lucene/docs/gettingstarted.pdf (original) and lucene/dev/branches/flexscoring/lucene/docs/gettingstarted.pdf Sat May 14 13:51:35 2011 differ

Modified: lucene/dev/branches/flexscoring/lucene/docs/index.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/docs/index.html?rev=1103112&r1=1103111&r2=1103112&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/docs/index.html (original)
+++ lucene/dev/branches/flexscoring/lucene/docs/index.html Sat May 14 13:51:35 2011
@@ -3,7 +3,7 @@
 <head>
 <META http-equiv="Content-Type" content="text/html; charset=UTF-8">
 <meta content="Apache Forrest" name="Generator">
-<meta name="Forrest-version" content="0.8">
+<meta name="Forrest-version" content="0.9">
 <meta name="Forrest-skin-name" content="lucene">
 <title>Lucene Java Documentation</title>
 <link type="text/css" href="skin/basic.css" rel="stylesheet">

Modified: lucene/dev/branches/flexscoring/lucene/docs/index.pdf
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/docs/index.pdf?rev=1103112&r1=1103111&r2=1103112&view=diff
==============================================================================
Files lucene/dev/branches/flexscoring/lucene/docs/index.pdf (original) and lucene/dev/branches/flexscoring/lucene/docs/index.pdf Sat May 14 13:51:35 2011 differ

Modified: lucene/dev/branches/flexscoring/lucene/docs/linkmap.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/docs/linkmap.html?rev=1103112&r1=1103111&r2=1103112&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/docs/linkmap.html (original)
+++ lucene/dev/branches/flexscoring/lucene/docs/linkmap.html Sat May 14 13:51:35 2011
@@ -3,7 +3,7 @@
 <head>
 <META http-equiv="Content-Type" content="text/html; charset=UTF-8">
 <meta content="Apache Forrest" name="Generator">
-<meta name="Forrest-version" content="0.8">
+<meta name="Forrest-version" content="0.9">
 <meta name="Forrest-skin-name" content="lucene">
 <title>Site Linkmap Table of Contents</title>
 <link type="text/css" href="skin/basic.css" rel="stylesheet">

Modified: lucene/dev/branches/flexscoring/lucene/docs/linkmap.pdf
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/docs/linkmap.pdf?rev=1103112&r1=1103111&r2=1103112&view=diff
==============================================================================
Files lucene/dev/branches/flexscoring/lucene/docs/linkmap.pdf (original) and lucene/dev/branches/flexscoring/lucene/docs/linkmap.pdf Sat May 14 13:51:35 2011 differ

Modified: lucene/dev/branches/flexscoring/lucene/docs/lucene-contrib/index.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/docs/lucene-contrib/index.html?rev=1103112&r1=1103111&r2=1103112&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/docs/lucene-contrib/index.html (original)
+++ lucene/dev/branches/flexscoring/lucene/docs/lucene-contrib/index.html Sat May 14 13:51:35 2011
@@ -3,7 +3,7 @@
 <head>
 <META http-equiv="Content-Type" content="text/html; charset=UTF-8">
 <meta content="Apache Forrest" name="Generator">
-<meta name="Forrest-version" content="0.8">
+<meta name="Forrest-version" content="0.9">
 <meta name="Forrest-skin-name" content="lucene">
 <title>
 	        Apache Lucene - Lucene Contrib

Modified: lucene/dev/branches/flexscoring/lucene/docs/lucene-contrib/index.pdf
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/docs/lucene-contrib/index.pdf?rev=1103112&r1=1103111&r2=1103112&view=diff
==============================================================================
Files lucene/dev/branches/flexscoring/lucene/docs/lucene-contrib/index.pdf (original) and lucene/dev/branches/flexscoring/lucene/docs/lucene-contrib/index.pdf Sat May 14 13:51:35 2011 differ

Modified: lucene/dev/branches/flexscoring/lucene/docs/queryparsersyntax.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/docs/queryparsersyntax.html?rev=1103112&r1=1103111&r2=1103112&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/docs/queryparsersyntax.html (original)
+++ lucene/dev/branches/flexscoring/lucene/docs/queryparsersyntax.html Sat May 14 13:51:35 2011
@@ -3,7 +3,7 @@
 <head>
 <META http-equiv="Content-Type" content="text/html; charset=UTF-8">
 <meta content="Apache Forrest" name="Generator">
-<meta name="Forrest-version" content="0.8">
+<meta name="Forrest-version" content="0.9">
 <meta name="Forrest-skin-name" content="lucene">
 <title>
 	Apache Lucene - Query Parser Syntax

Modified: lucene/dev/branches/flexscoring/lucene/docs/queryparsersyntax.pdf
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/docs/queryparsersyntax.pdf?rev=1103112&r1=1103111&r2=1103112&view=diff
==============================================================================
Files lucene/dev/branches/flexscoring/lucene/docs/queryparsersyntax.pdf (original) and lucene/dev/branches/flexscoring/lucene/docs/queryparsersyntax.pdf Sat May 14 13:51:35 2011 differ

Modified: lucene/dev/branches/flexscoring/lucene/docs/scoring.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/docs/scoring.html?rev=1103112&r1=1103111&r2=1103112&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/docs/scoring.html (original)
+++ lucene/dev/branches/flexscoring/lucene/docs/scoring.html Sat May 14 13:51:35 2011
@@ -3,7 +3,7 @@
 <head>
 <META http-equiv="Content-Type" content="text/html; charset=UTF-8">
 <meta content="Apache Forrest" name="Generator">
-<meta name="Forrest-version" content="0.8">
+<meta name="Forrest-version" content="0.9">
 <meta name="Forrest-skin-name" content="lucene">
 <title>
 	Apache Lucene - Scoring

Modified: lucene/dev/branches/flexscoring/lucene/docs/scoring.pdf
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/docs/scoring.pdf?rev=1103112&r1=1103111&r2=1103112&view=diff
==============================================================================
Files lucene/dev/branches/flexscoring/lucene/docs/scoring.pdf (original) and lucene/dev/branches/flexscoring/lucene/docs/scoring.pdf Sat May 14 13:51:35 2011 differ

Modified: lucene/dev/branches/flexscoring/lucene/docs/systemrequirements.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/docs/systemrequirements.html?rev=1103112&r1=1103111&r2=1103112&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/docs/systemrequirements.html (original)
+++ lucene/dev/branches/flexscoring/lucene/docs/systemrequirements.html Sat May 14 13:51:35 2011
@@ -3,7 +3,7 @@
 <head>
 <META http-equiv="Content-Type" content="text/html; charset=UTF-8">
 <meta content="Apache Forrest" name="Generator">
-<meta name="Forrest-version" content="0.8">
+<meta name="Forrest-version" content="0.9">
 <meta name="Forrest-skin-name" content="lucene">
 <title>Apache Lucene - System Requirements</title>
 <link type="text/css" href="skin/basic.css" rel="stylesheet">

Modified: lucene/dev/branches/flexscoring/lucene/docs/systemrequirements.pdf
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/docs/systemrequirements.pdf?rev=1103112&r1=1103111&r2=1103112&view=diff
==============================================================================
Binary files - no diff available.

Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/document/Document.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/document/Document.java?rev=1103112&r1=1103111&r2=1103112&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/document/Document.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/document/Document.java Sat May 14 13:51:35 2011
@@ -131,8 +131,13 @@ public final class Document {
   /** Returns a field with the given name if any exist in this document, or
    * null.  If multiple fields exists with this name, this method returns the
    * first value added.
-   * Do not use this method with lazy loaded fields.
+   * Do not use this method with lazy loaded fields or {@link NumericField}.
+   * @deprecated use {@link #getFieldable} instead and cast depending on
+   * data type.
+   * @throws ClassCastException if you try to retrieve a numerical or
+   * lazy loaded field.
    */
+  @Deprecated
   public final Field getField(String name) {
     return (Field) getFieldable(name);
   }
@@ -154,6 +159,8 @@ public final class Document {
    * this document, or null.  If multiple fields exist with this name, this
    * method returns the first value added. If only binary fields with this name
    * exist, returns null.
+   * For {@link NumericField} it returns the string value of the number. If you want
+   * the actual {@code NumericField} instance back, use {@link #getFieldable}.
    */
   public final String get(String name) {
    for (Fieldable field : fields) {
@@ -177,13 +184,18 @@ public final class Document {
   
   /**
    * Returns an array of {@link Field}s with the given name.
-   * Do not use with lazy loaded fields.
    * This method returns an empty array when there are no
    * matching fields.  It never returns null.
+   * Do not use this method with lazy loaded fields or {@link NumericField}.
    *
    * @param name the name of the field
    * @return a <code>Field[]</code> array
+   * @deprecated use {@link #getFieldable} instead and cast depending on
+   * data type.
+   * @throws ClassCastException if you try to retrieve a numerical or
+   * lazy loaded field.
    */
+   @Deprecated
    public final Field[] getFields(String name) {
      List<Field> result = new ArrayList<Field>();
      for (Fieldable field : fields) {
@@ -230,6 +242,8 @@ public final class Document {
    * Returns an array of values of the field specified as the method parameter.
    * This method returns an empty array when there are no
    * matching fields.  It never returns null.
+   * For {@link NumericField}s it returns the string value of the number. If you want
+   * the actual {@code NumericField} instances back, use {@link #getFieldables}.
    * @param name the name of the field
    * @return a <code>String[]</code> of field values
    */

Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/document/NumericField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/document/NumericField.java?rev=1103112&r1=1103111&r2=1103112&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/document/NumericField.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/document/NumericField.java Sat May 14 13:51:35 2011
@@ -127,18 +127,18 @@ import org.apache.lucene.search.FieldCac
  * class is a wrapper around this token stream type for
  * easier, more intuitive usage.</p>
  *
- * <p><b>NOTE:</b> This class is only used during
- * indexing. When retrieving the stored field value from a
- * {@link Document} instance after search, you will get a
- * conventional {@link Fieldable} instance where the numeric
- * values are returned as {@link String}s (according to
- * <code>toString(value)</code> of the used data type).
- *
  * @since 2.9
  */
 public final class NumericField extends AbstractField {
 
-  private final NumericTokenStream numericTS;
+  /** Data type of the value in {@link NumericField}.
+   * @since 3.2
+   */
+  public static enum DataType { INT, LONG, FLOAT, DOUBLE }
+
+  private transient NumericTokenStream numericTS;
+  private DataType type;
+  private final int precisionStep;
 
   /**
    * Creates a field for numeric values using the default <code>precisionStep</code>
@@ -158,8 +158,8 @@ public final class NumericField extends 
    * a numeric value, before indexing a document containing this field,
    * set a value using the various set<em>???</em>Value() methods.
    * @param name the field name
-   * @param store if the field should be stored in plain text form
-   *  (according to <code>toString(value)</code> of the used data type)
+   * @param store if the field should be stored, {@link Document#getFieldable}
+   * then returns {@code NumericField} instances on search results.
    * @param index if the field should be indexed using {@link NumericTokenStream}
    */
   public NumericField(String name, Field.Store store, boolean index) {
@@ -186,19 +186,43 @@ public final class NumericField extends 
    * set a value using the various set<em>???</em>Value() methods.
    * @param name the field name
    * @param precisionStep the used <a href="../search/NumericRangeQuery.html#precisionStepDesc">precision step</a>
-   * @param store if the field should be stored in plain text form
-   *  (according to <code>toString(value)</code> of the used data type)
+   * @param store if the field should be stored, {@link Document#getFieldable}
+   * then returns {@code NumericField} instances on search results.
    * @param index if the field should be indexed using {@link NumericTokenStream}
    */
   public NumericField(String name, int precisionStep, Field.Store store, boolean index) {
     super(name, store, index ? Field.Index.ANALYZED_NO_NORMS : Field.Index.NO, Field.TermVector.NO);
+    this.precisionStep = precisionStep;
     setOmitTermFreqAndPositions(true);
-    numericTS = new NumericTokenStream(precisionStep);
   }
 
   /** Returns a {@link NumericTokenStream} for indexing the numeric value. */
   public TokenStream tokenStreamValue()   {
-    return isIndexed() ? numericTS : null;
+    if (!isIndexed())
+      return null;
+    if (numericTS == null) {
+      // lazy init the TokenStream as it is heavy to instantiate (attributes,...),
+      // if not needed (stored field loading)
+      numericTS = new NumericTokenStream(precisionStep);
+      // initialize value in TokenStream
+      if (fieldsData != null) {
+        assert type != null;
+        final Number val = (Number) fieldsData;
+        switch (type) {
+          case INT:
+            numericTS.setIntValue(val.intValue()); break;
+          case LONG:
+            numericTS.setLongValue(val.longValue()); break;
+          case FLOAT:
+            numericTS.setFloatValue(val.floatValue()); break;
+          case DOUBLE:
+            numericTS.setDoubleValue(val.doubleValue()); break;
+          default:
+            assert false : "Should never get here";
+        }
+      }
+    }
+    return numericTS;
   }
   
   /** Returns always <code>null</code> for numeric fields */
@@ -212,7 +236,10 @@ public final class NumericField extends 
     return null;
   }
     
-  /** Returns the numeric value as a string (how it is stored, when {@link Field.Store#YES} is chosen). */
+  /** Returns the numeric value as a string. This format is also returned if you call {@link Document#get(String)}
+   * on search results. It is recommended to use {@link Document#getFieldable} instead
+   * that returns {@code NumericField} instances. You can then use {@link #getNumericValue}
+   * to return the stored value. */
   public String stringValue()   {
     return (fieldsData == null) ? null : fieldsData.toString();
   }
@@ -224,7 +251,14 @@ public final class NumericField extends 
   
   /** Returns the precision step. */
   public int getPrecisionStep() {
-    return numericTS.getPrecisionStep();
+    return precisionStep;
+  }
+  
+  /** Returns the data type of the current value, {@code null} if not yet set.
+   * @since 3.2
+   */
+  public DataType getDataType() {
+    return type;
   }
   
   /**
@@ -234,8 +268,9 @@ public final class NumericField extends 
    * <code>document.add(new NumericField(name, precisionStep).setLongValue(value))</code>
    */
   public NumericField setLongValue(final long value) {
-    numericTS.setLongValue(value);
+    if (numericTS != null) numericTS.setLongValue(value);
     fieldsData = Long.valueOf(value);
+    type = DataType.LONG;
     return this;
   }
   
@@ -246,8 +281,9 @@ public final class NumericField extends 
    * <code>document.add(new NumericField(name, precisionStep).setIntValue(value))</code>
    */
   public NumericField setIntValue(final int value) {
-    numericTS.setIntValue(value);
+    if (numericTS != null) numericTS.setIntValue(value);
     fieldsData = Integer.valueOf(value);
+    type = DataType.INT;
     return this;
   }
   
@@ -258,8 +294,9 @@ public final class NumericField extends 
    * <code>document.add(new NumericField(name, precisionStep).setDoubleValue(value))</code>
    */
   public NumericField setDoubleValue(final double value) {
-    numericTS.setDoubleValue(value);
+    if (numericTS != null) numericTS.setDoubleValue(value);
     fieldsData = Double.valueOf(value);
+    type = DataType.DOUBLE;
     return this;
   }
   
@@ -270,8 +307,9 @@ public final class NumericField extends 
    * <code>document.add(new NumericField(name, precisionStep).setFloatValue(value))</code>
    */
   public NumericField setFloatValue(final float value) {
-    numericTS.setFloatValue(value);
+    if (numericTS != null) numericTS.setFloatValue(value);
     fieldsData = Float.valueOf(value);
+    type = DataType.FLOAT;
     return this;
   }
 

Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/ByteSliceWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/ByteSliceWriter.java?rev=1103112&r1=1103111&r2=1103112&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/ByteSliceWriter.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/ByteSliceWriter.java Sat May 14 13:51:35 2011
@@ -81,6 +81,6 @@ final class ByteSliceWriter extends Data
   }
 
   public int getAddress() {
-    return upto + (offset0 & DocumentsWriter.BYTE_BLOCK_NOT_MASK);
+    return upto + (offset0 & DocumentsWriterPerThread.BYTE_BLOCK_NOT_MASK);
   }
 }
\ No newline at end of file

Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/CheckIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/CheckIndex.java?rev=1103112&r1=1103111&r2=1103112&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/CheckIndex.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/CheckIndex.java Sat May 14 13:51:35 2011
@@ -661,10 +661,13 @@ public class CheckIndex {
           status.termCount++;
 
           final DocsEnum docs2;
+          final boolean hasPositions;
           if (postings != null) {
             docs2 = postings;
+            hasPositions = true;
           } else {
             docs2 = docs;
+            hasPositions = false;
           }
 
           int lastDoc = -1;
@@ -733,6 +736,67 @@ public class CheckIndex {
               throw new RuntimeException("term " + term + " totalTermFreq=" + totalTermFreq2 + " != recomputed totalTermFreq=" + totalTermFreq);
             }
           }
+
+          // Test skipping
+          if (docFreq >= 16) {
+            if (hasPositions) {
+              for(int idx=0;idx<7;idx++) {
+                final int skipDocID = (int) (((idx+1)*(long) maxDoc)/8);
+                postings = terms.docsAndPositions(delDocs, postings);
+                final int docID = postings.advance(skipDocID);
+                if (docID == DocsEnum.NO_MORE_DOCS) {
+                  break;
+                } else {
+                  if (docID < skipDocID) {
+                    throw new RuntimeException("term " + term + ": advance(docID=" + skipDocID + ") returned docID=" + docID);
+                  }
+                  final int freq = postings.freq();
+                  if (freq <= 0) {
+                    throw new RuntimeException("termFreq " + freq + " is out of bounds");
+                  }
+                  int lastPosition = -1;
+                  for(int posUpto=0;posUpto<freq;posUpto++) {
+                    final int pos = postings.nextPosition();
+                    if (pos < 0) {
+                      throw new RuntimeException("position " + pos + " is out of bounds");
+                    }
+                    if (pos <= lastPosition) {
+                      throw new RuntimeException("position " + pos + " is <= lastPosition " + lastPosition);
+                    }
+                    lastPosition = pos;
+                  } 
+
+                  final int nextDocID = postings.nextDoc();
+                  if (nextDocID == DocsEnum.NO_MORE_DOCS) {
+                    break;
+                  }
+                  if (nextDocID <= docID) {
+                    throw new RuntimeException("term " + term + ": advance(docID=" + skipDocID + "), then .next() returned docID=" + nextDocID + " vs prev docID=" + docID);
+                  }
+                }
+              }
+            } else {
+              for(int idx=0;idx<7;idx++) {
+                final int skipDocID = (int) (((idx+1)*(long) maxDoc)/8);
+                docs = terms.docs(delDocs, docs);
+                final int docID = docs.advance(skipDocID);
+                if (docID == DocsEnum.NO_MORE_DOCS) {
+                  break;
+                } else {
+                  if (docID < skipDocID) {
+                    throw new RuntimeException("term " + term + ": advance(docID=" + skipDocID + ") returned docID=" + docID);
+                  }
+                  final int nextDocID = docs.nextDoc();
+                  if (nextDocID == DocsEnum.NO_MORE_DOCS) {
+                    break;
+                  }
+                  if (nextDocID <= docID) {
+                    throw new RuntimeException("term " + term + ": advance(docID=" + skipDocID + "), then .next() returned docID=" + nextDocID + " vs prev docID=" + docID);
+                  }
+                }
+              }
+            }
+          }
         }
 
         if (sumTotalTermFreq != 0) {

Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/CompoundFileWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/CompoundFileWriter.java?rev=1103112&r1=1103111&r2=1103112&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/CompoundFileWriter.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/CompoundFileWriter.java Sat May 14 13:51:35 2011
@@ -46,8 +46,10 @@ import org.apache.lucene.util.IOUtils;
  * file. The {directory} that follows has that many entries. Each directory entry
  * contains a long pointer to the start of this file's data section, and a String
  * with that file's name.
+ * 
+ * @lucene.internal
  */
-final class CompoundFileWriter {
+public final class CompoundFileWriter {
 
     private static final class FileEntry {
         /** source file */
@@ -136,8 +138,7 @@ final class CompoundFileWriter {
 
     /** Merge files with the extensions added up to now.
      *  All files with these extensions are combined sequentially into the
-     *  compound stream. After successful merge, the source files
-     *  are deleted.
+     *  compound stream.
      *  @throws IllegalStateException if close() had been called before or
      *   if no file has been added to this object
      */

Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java?rev=1103112&r1=1103111&r2=1103112&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java Sat May 14 13:51:35 2011
@@ -135,8 +135,8 @@ public class ConcurrentMergeScheduler ex
       final MergePolicy.OneMerge m1 = t1.getCurrentMerge();
       final MergePolicy.OneMerge m2 = t2.getCurrentMerge();
       
-      final int c1 = m1 == null ? Integer.MAX_VALUE : m1.segments.totalDocCount();
-      final int c2 = m2 == null ? Integer.MAX_VALUE : m2.segments.totalDocCount();
+      final int c1 = m1 == null ? Integer.MAX_VALUE : m1.totalDocCount;
+      final int c2 = m2 == null ? Integer.MAX_VALUE : m2.totalDocCount;
 
       return c2 - c1;
     }

Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/DocConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/DocConsumer.java?rev=1103112&r1=1103111&r2=1103112&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/DocConsumer.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/DocConsumer.java Sat May 14 13:51:35 2011
@@ -18,11 +18,12 @@ package org.apache.lucene.index;
  */
 
 import java.io.IOException;
-import java.util.Collection;
 
 abstract class DocConsumer {
-  abstract DocConsumerPerThread addThread(DocumentsWriterThreadState perThread) throws IOException;
-  abstract void flush(final Collection<DocConsumerPerThread> threads, final SegmentWriteState state) throws IOException;
+  abstract void processDocument(FieldInfos fieldInfos) throws IOException;
+  abstract void finishDocument() throws IOException;
+  abstract void flush(final SegmentWriteState state) throws IOException;
   abstract void abort();
   abstract boolean freeRAM();
+  abstract void doAfterFlush();
 }

Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/DocFieldConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/DocFieldConsumer.java?rev=1103112&r1=1103111&r2=1103112&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/DocFieldConsumer.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/DocFieldConsumer.java Sat May 14 13:51:35 2011
@@ -18,22 +18,25 @@ package org.apache.lucene.index;
  */
 
 import java.io.IOException;
-import java.util.Collection;
 import java.util.Map;
 
 abstract class DocFieldConsumer {
-  /** Called when DocumentsWriter decides to create a new
+  /** Called when DocumentsWriterPerThread decides to create a new
    *  segment */
-  abstract void flush(Map<DocFieldConsumerPerThread,Collection<DocFieldConsumerPerField>> threadsAndFields, SegmentWriteState state) throws IOException;
+  abstract void flush(Map<FieldInfo, DocFieldConsumerPerField> fieldsToFlush, SegmentWriteState state) throws IOException;
 
   /** Called when an aborting exception is hit */
   abstract void abort();
 
-  /** Add a new thread */
-  abstract DocFieldConsumerPerThread addThread(DocFieldProcessorPerThread docFieldProcessorPerThread) throws IOException;
-
-  /** Called when DocumentsWriter is using too much RAM.
+  /** Called when DocumentsWriterPerThread is using too much RAM.
    *  The consumer should free RAM, if possible, returning
    *  true if any RAM was in fact freed. */
   abstract boolean freeRAM();
-  }
+
+  abstract void startDocument() throws IOException;
+
+  abstract DocFieldConsumerPerField addField(FieldInfo fi);
+
+  abstract void finishDocument() throws IOException;
+
+}

Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/DocFieldConsumerPerField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/DocFieldConsumerPerField.java?rev=1103112&r1=1103111&r2=1103112&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/DocFieldConsumerPerField.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/DocFieldConsumerPerField.java Sat May 14 13:51:35 2011
@@ -24,4 +24,5 @@ abstract class DocFieldConsumerPerField 
   /** Processes all occurrences of a single field */
   abstract void processFields(Fieldable[] fields, int count) throws IOException;
   abstract void abort();
+  abstract FieldInfo getFieldInfo();
 }



Mime
View raw message