lucene-java-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From uschind...@apache.org
Subject svn commit: r931278 [2/10] - in /lucene/dev/trunk: lucene/ lucene/backwards/src/ lucene/backwards/src/java/org/apache/lucene/index/ lucene/backwards/src/java/org/apache/lucene/index/codecs/ lucene/backwards/src/java/org/apache/lucene/search/ lucene/bac...
Date Tue, 06 Apr 2010 19:19:36 GMT
Modified: lucene/dev/trunk/lucene/backwards/src/java/org/apache/lucene/store/IndexOutput.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/backwards/src/java/org/apache/lucene/store/IndexOutput.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/backwards/src/java/org/apache/lucene/store/IndexOutput.java (original)
+++ lucene/dev/trunk/lucene/backwards/src/java/org/apache/lucene/store/IndexOutput.java Tue Apr  6 19:19:27 2010
@@ -17,166 +17,15 @@ package org.apache.lucene.store;
  * limitations under the License.
  */
 
-import java.io.IOException;
 import java.io.Closeable;
-import java.util.Map;
-import org.apache.lucene.util.UnicodeUtil;
+import java.io.IOException;
 
 /** Abstract base class for output to a file in a Directory.  A random-access
  * output stream.  Used for all Lucene index output operations.
  * @see Directory
  * @see IndexInput
  */
-public abstract class IndexOutput implements Closeable {
-
-  private UnicodeUtil.UTF8Result utf8Result = new UnicodeUtil.UTF8Result();
-
-  /** Writes a single byte.
-   * @see IndexInput#readByte()
-   */
-  public abstract void writeByte(byte b) throws IOException;
-
-  /** Writes an array of bytes.
-   * @param b the bytes to write
-   * @param length the number of bytes to write
-   * @see IndexInput#readBytes(byte[],int,int)
-   */
-  public void writeBytes(byte[] b, int length) throws IOException {
-    writeBytes(b, 0, length);
-  }
-
-  /** Writes an array of bytes.
-   * @param b the bytes to write
-   * @param offset the offset in the byte array
-   * @param length the number of bytes to write
-   * @see IndexInput#readBytes(byte[],int,int)
-   */
-  public abstract void writeBytes(byte[] b, int offset, int length) throws IOException;
-
-  /** Writes an int as four bytes.
-   * @see IndexInput#readInt()
-   */
-  public void writeInt(int i) throws IOException {
-    writeByte((byte)(i >> 24));
-    writeByte((byte)(i >> 16));
-    writeByte((byte)(i >>  8));
-    writeByte((byte) i);
-  }
-
-  /** Writes an int in a variable-length format.  Writes between one and
-   * five bytes.  Smaller values take fewer bytes.  Negative numbers are not
-   * supported.
-   * @see IndexInput#readVInt()
-   */
-  public void writeVInt(int i) throws IOException {
-    while ((i & ~0x7F) != 0) {
-      writeByte((byte)((i & 0x7f) | 0x80));
-      i >>>= 7;
-    }
-    writeByte((byte)i);
-  }
-
-  /** Writes a long as eight bytes.
-   * @see IndexInput#readLong()
-   */
-  public void writeLong(long i) throws IOException {
-    writeInt((int) (i >> 32));
-    writeInt((int) i);
-  }
-
-  /** Writes an long in a variable-length format.  Writes between one and five
-   * bytes.  Smaller values take fewer bytes.  Negative numbers are not
-   * supported.
-   * @see IndexInput#readVLong()
-   */
-  public void writeVLong(long i) throws IOException {
-    while ((i & ~0x7F) != 0) {
-      writeByte((byte)((i & 0x7f) | 0x80));
-      i >>>= 7;
-    }
-    writeByte((byte)i);
-  }
-
-  /** Writes a string.
-   * @see IndexInput#readString()
-   */
-  public void writeString(String s) throws IOException {
-    UnicodeUtil.UTF16toUTF8(s, 0, s.length(), utf8Result);
-    writeVInt(utf8Result.length);
-    writeBytes(utf8Result.result, 0, utf8Result.length);
-  }
-
-  /** Writes a sub sequence of characters from s as the old
-   *  format (modified UTF-8 encoded bytes).
-   * @param s the source of the characters
-   * @param start the first character in the sequence
-   * @param length the number of characters in the sequence
-   * @deprecated -- please pre-convert to utf8 bytes
-   * instead or use {@link #writeString}
-   */
-  public void writeChars(String s, int start, int length)
-       throws IOException {
-    final int end = start + length;
-    for (int i = start; i < end; i++) {
-      final int code = (int)s.charAt(i);
-      if (code >= 0x01 && code <= 0x7F)
-	writeByte((byte)code);
-      else if (((code >= 0x80) && (code <= 0x7FF)) || code == 0) {
-	writeByte((byte)(0xC0 | (code >> 6)));
-	writeByte((byte)(0x80 | (code & 0x3F)));
-      } else {
-	writeByte((byte)(0xE0 | (code >>> 12)));
-	writeByte((byte)(0x80 | ((code >> 6) & 0x3F)));
-	writeByte((byte)(0x80 | (code & 0x3F)));
-      }
-    }
-  }
-
-  /** Writes a sub sequence of characters from char[] as
-   *  the old format (modified UTF-8 encoded bytes).
-   * @param s the source of the characters
-   * @param start the first character in the sequence
-   * @param length the number of characters in the sequence
-   * @deprecated -- please pre-convert to utf8 bytes instead or use {@link #writeString}
-   */
-  public void writeChars(char[] s, int start, int length)
-    throws IOException {
-    final int end = start + length;
-    for (int i = start; i < end; i++) {
-      final int code = (int)s[i];
-      if (code >= 0x01 && code <= 0x7F)
-	writeByte((byte)code);
-      else if (((code >= 0x80) && (code <= 0x7FF)) || code == 0) {
-	writeByte((byte)(0xC0 | (code >> 6)));
-	writeByte((byte)(0x80 | (code & 0x3F)));
-      } else {
-	writeByte((byte)(0xE0 | (code >>> 12)));
-	writeByte((byte)(0x80 | ((code >> 6) & 0x3F)));
-	writeByte((byte)(0x80 | (code & 0x3F)));
-      }
-    }
-  }
-
-  private static int COPY_BUFFER_SIZE = 16384;
-  private byte[] copyBuffer;
-
-  /** Copy numBytes bytes from input to ourself. */
-  public void copyBytes(IndexInput input, long numBytes) throws IOException {
-    assert numBytes >= 0: "numBytes=" + numBytes;
-    long left = numBytes;
-    if (copyBuffer == null)
-      copyBuffer = new byte[COPY_BUFFER_SIZE];
-    while(left > 0) {
-      final int toCopy;
-      if (left > COPY_BUFFER_SIZE)
-        toCopy = COPY_BUFFER_SIZE;
-      else
-        toCopy = (int) left;
-      input.readBytes(copyBuffer, 0, toCopy);
-      writeBytes(copyBuffer, 0, toCopy);
-      left -= toCopy;
-    }
-  }
+public abstract class IndexOutput extends DataOutput implements Closeable {
 
   /** Forces any buffered output to be written. */
   public abstract void flush() throws IOException;
@@ -208,17 +57,5 @@ public abstract class IndexOutput implem
    * undefined.  Otherwise the file is truncated.
    * @param length file length
    */
-  public void setLength(long length) throws IOException {};
-
-  public void writeStringStringMap(Map<String,String> map) throws IOException {
-    if (map == null) {
-      writeInt(0);
-    } else {
-      writeInt(map.size());
-      for(final Map.Entry<String, String> entry: map.entrySet()) {
-        writeString(entry.getKey());
-        writeString(entry.getValue());
-      }
-    }
-  }
+  public void setLength(long length) throws IOException {}
 }

Modified: lucene/dev/trunk/lucene/backwards/src/java/org/apache/lucene/util/UnicodeUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/backwards/src/java/org/apache/lucene/util/UnicodeUtil.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/backwards/src/java/org/apache/lucene/util/UnicodeUtil.java (original)
+++ lucene/dev/trunk/lucene/backwards/src/java/org/apache/lucene/util/UnicodeUtil.java Tue Apr  6 19:19:27 2010
@@ -106,6 +106,10 @@ final public class UnicodeUtil {
     }
   }
 
+  // stubs for tests only
+  public static void UTF16toUTF8(char[] source, int offset, int length, BytesRef result) {}
+  public static void UTF16toUTF8(CharSequence s, int offset, int length, BytesRef result) {}
+
   /** Encode characters from a char[] source, starting at
    *  offset and stopping when the character 0xffff is seen.
    *  Returns the number of bytes written to bytesOut. */
@@ -223,7 +227,7 @@ final public class UnicodeUtil {
   /** Encode characters from this String, starting at offset
    *  for length characters.  Returns the number of bytes
    *  written to bytesOut. */
-  public static void UTF16toUTF8(final String s, final int offset, final int length, UTF8Result result) {
+  public static void UTF16toUTF8(final CharSequence s, final int offset, final int length, UTF8Result result) {
     final int end = offset + length;
 
     byte[] out = result.result;

Propchange: lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Apr  6 19:19:27 2010
@@ -1,5 +1,7 @@
+/lucene/java/branches/flex_1458/backwards/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java:824912-931101
 /lucene/java/branches/lucene_2_4/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java:748824
 /lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java:825998,829134,829881,831036
 /lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java:880754,880793,880823,881216,881317,881376,881473,881549,881820,882467,882890,883076,883080,912383
-/lucene/java/trunk/backwards/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java:924483-925561
+/lucene/java/branches/lucene_3_0_back_compat_tests/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java:880869-912395
+/lucene/java/trunk/backwards/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java:924483-924731,924781,925176-925462
 /lucene/java/trunk/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java:881213,881315,881466,881819,882374,882807,882888,882977,883074-883075,885214,912319,912324,912328,912335,912372

Modified: lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/analysis/tokenattributes/TestTermAttributeImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/analysis/tokenattributes/TestTermAttributeImpl.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/analysis/tokenattributes/TestTermAttributeImpl.java (original)
+++ lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/analysis/tokenattributes/TestTermAttributeImpl.java Tue Apr  6 19:19:27 2010
@@ -107,10 +107,10 @@ public class TestTermAttributeImpl exten
     char[] b = {'a', 'l', 'o', 'h', 'a'};
     TermAttributeImpl t = new TermAttributeImpl();
     t.setTermBuffer(b, 0, 5);
-    assertEquals("term=aloha", t.toString());
+    assertEquals("aloha", t.toString());
 
     t.setTermBuffer("hi there");
-    assertEquals("term=hi there", t.toString());
+    assertEquals("hi there", t.toString());
   }
 
   public void testMixedStringArray() throws Exception {

Propchange: lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/document/TestDateTools.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Apr  6 19:19:27 2010
@@ -1,5 +1,7 @@
+/lucene/java/branches/flex_1458/backwards/src/test/org/apache/lucene/document/TestDateTools.java:824912-931101
 /lucene/java/branches/lucene_2_4/src/test/org/apache/lucene/document/TestDateTools.java:748824
 /lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/document/TestDateTools.java:825998,829134,829881,831036
 /lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/document/TestDateTools.java:880754,880793,880823,881216,881317,881376,881473,881549,881820,882467,882890,883076,883080,912383
-/lucene/java/trunk/backwards/src/test/org/apache/lucene/document/TestDateTools.java:924483-925561
+/lucene/java/branches/lucene_3_0_back_compat_tests/src/test/org/apache/lucene/document/TestDateTools.java:880869-912395
+/lucene/java/trunk/backwards/src/test/org/apache/lucene/document/TestDateTools.java:924483-924731,924781,925176-925462
 /lucene/java/trunk/src/test/org/apache/lucene/document/TestDateTools.java:881213,881315,881466,881819,882374,882807,882888,882977,883074-883075,885214,912319,912324,912328,912335,912372

Propchange: lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/document/TestNumberTools.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Apr  6 19:19:27 2010
@@ -1,5 +1,7 @@
+/lucene/java/branches/flex_1458/backwards/src/test/org/apache/lucene/document/TestNumberTools.java:824912-931101
 /lucene/java/branches/lucene_2_4/src/test/org/apache/lucene/document/TestNumberTools.java:748824
 /lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/document/TestNumberTools.java:825998,829134,829881,831036
 /lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/document/TestNumberTools.java:880754,880793,880823,881216,881317,881376,881473,881549,881820,882467,882890,883076,883080,912383
-/lucene/java/trunk/backwards/src/test/org/apache/lucene/document/TestNumberTools.java:924483-925561
+/lucene/java/branches/lucene_3_0_back_compat_tests/src/test/org/apache/lucene/document/TestNumberTools.java:880869-912395
+/lucene/java/trunk/backwards/src/test/org/apache/lucene/document/TestNumberTools.java:924483-924731,924781,925176-925462
 /lucene/java/trunk/src/test/org/apache/lucene/document/TestNumberTools.java:881213,881315,881466,881819,882374,882807,882888,882977,883074-883075,885214,912319,912324,912328,912335,912372

Propchange: lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Apr  6 19:19:27 2010
@@ -1,5 +1,7 @@
+/lucene/java/branches/flex_1458/backwards/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:824912-931101
 /lucene/java/branches/lucene_2_4/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:748824
 /lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:825998,829134,829881,831036
 /lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:880754,880793,880823,881216,881317,881376,881473,881549,881820,882467,882890,883076,883080,912383
-/lucene/java/trunk/backwards/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:924483-925561
+/lucene/java/branches/lucene_3_0_back_compat_tests/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:880869-912395
+/lucene/java/trunk/backwards/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:924483-924731,924781,925176-925462
 /lucene/java/trunk/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:881213,881315,881466,881819,882374,882807,882888,882977,883074-883075,885214,912319,912324,912328,912335,912372

Modified: lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/index/TestDoc.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/index/TestDoc.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/index/TestDoc.java (original)
+++ lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/index/TestDoc.java Tue Apr  6 19:19:27 2010
@@ -35,6 +35,7 @@ import org.apache.lucene.document.Field;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.FSDirectory;
 import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.index.codecs.CodecProvider;
 
 
 /** JUnit adaptation of an older test case DocTest. */
@@ -180,20 +181,24 @@ public class TestDoc extends LuceneTestC
       SegmentReader r1 = SegmentReader.get(true, si1, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR);
       SegmentReader r2 = SegmentReader.get(true, si2, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR);
 
-      SegmentMerger merger = new SegmentMerger(si1.dir, merged);
+      SegmentMerger merger = new SegmentMerger(si1.dir, IndexWriter.DEFAULT_TERM_INDEX_INTERVAL, merged, null, CodecProvider.getDefault());
 
       merger.add(r1);
       merger.add(r2);
       merger.merge();
       merger.closeReaders();
       
+      final SegmentInfo info = new SegmentInfo(merged, si1.docCount + si2.docCount, si1.dir,
+                                               useCompoundFile, true, -1, null, false, merger.hasProx(),
+                                               merger.getCodec());
+      
       if (useCompoundFile) {
-        List filesToDelete = merger.createCompoundFile(merged + ".cfs");
+        List filesToDelete = merger.createCompoundFile(merged + ".cfs", info);
         for (Iterator iter = filesToDelete.iterator(); iter.hasNext();)
           si1.dir.deleteFile((String) iter.next());
       }
 
-      return new SegmentInfo(merged, si1.docCount + si2.docCount, si1.dir, useCompoundFile, true);
+      return info;
    }
 
 

Modified: lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/index/TestIndexReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/index/TestIndexReader.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/index/TestIndexReader.java (original)
+++ lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/index/TestIndexReader.java Tue Apr  6 19:19:27 2010
@@ -986,29 +986,7 @@ public class TestIndexReader extends Luc
           // new IndexFileDeleter, have it delete
           // unreferenced files, then verify that in fact
           // no files were deleted:
-          String[] startFiles = dir.listAll();
-          SegmentInfos infos = new SegmentInfos();
-          infos.read(dir);
-          new IndexFileDeleter(dir, new KeepOnlyLastCommitDeletionPolicy(), infos, null, null);
-          String[] endFiles = dir.listAll();
-
-          Arrays.sort(startFiles);
-          Arrays.sort(endFiles);
-
-          //for(int i=0;i<startFiles.length;i++) {
-          //  System.out.println("  startFiles: " + i + ": " + startFiles[i]);
-          //}
-
-          if (!Arrays.equals(startFiles, endFiles)) {
-            String successStr;
-            if (success) {
-              successStr = "success";
-            } else {
-              successStr = "IOException";
-              err.printStackTrace();
-            }
-            fail("reader.close() failed to delete unreferenced files after " + successStr + " (" + diskFree + " bytes): before delete:\n    " + arrayToString(startFiles) + "\n  after delete:\n    " + arrayToString(endFiles));
-          }
+          TestIndexWriter.assertNoUnreferencedFiles(dir, "reader.close() failed to delete unreferenced files");
 
           // Finally, verify index is not corrupt, and, if
           // we succeeded, we see all docs changed, and if
@@ -1760,7 +1738,6 @@ public class TestIndexReader extends Luc
     } catch (IllegalStateException ise) {
       // expected
     }
-    assertFalse(((SegmentReader) r.getSequentialSubReaders()[0]).termsIndexLoaded());
 
     assertEquals(-1, ((SegmentReader) r.getSequentialSubReaders()[0]).getTermInfosIndexDivisor());
     writer = new IndexWriter(dir, new WhitespaceAnalyzer(), IndexWriter.MaxFieldLength.UNLIMITED);
@@ -1773,7 +1750,12 @@ public class TestIndexReader extends Luc
     IndexReader[] subReaders = r2.getSequentialSubReaders();
     assertEquals(2, subReaders.length);
     for(int i=0;i<2;i++) {
-      assertFalse(((SegmentReader) subReaders[i]).termsIndexLoaded());
+      try {
+        subReaders[i].docFreq(new Term("field", "f"));
+        fail("did not hit expected exception");
+      } catch (IllegalStateException ise) {
+        // expected
+      }
     }
     r2.close();
     dir.close();

Modified: lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/index/TestIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/index/TestIndexWriter.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/index/TestIndexWriter.java (original)
+++ lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/index/TestIndexWriter.java Tue Apr  6 19:19:27 2010
@@ -61,8 +61,10 @@ import org.apache.lucene.store.IndexOutp
 import org.apache.lucene.store.Lock;
 import org.apache.lucene.store.LockFactory;
 import org.apache.lucene.store.MockRAMDirectory;
+import org.apache.lucene.store.NoLockFactory;
 import org.apache.lucene.store.RAMDirectory;
 import org.apache.lucene.store.SingleInstanceLockFactory;
+import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.UnicodeUtil;
 import org.apache.lucene.util._TestUtil;
 import org.apache.lucene.util.Version;
@@ -524,10 +526,15 @@ public class TestIndexWriter extends Luc
     }                                               
 
     public static void assertNoUnreferencedFiles(Directory dir, String message) throws IOException {
-      String[] startFiles = dir.listAll();
-      SegmentInfos infos = new SegmentInfos();
-      infos.read(dir);
-      new IndexFileDeleter(dir, new KeepOnlyLastCommitDeletionPolicy(), infos, null, null);
+      final LockFactory lf = dir.getLockFactory();
+      String[] startFiles;
+      try {
+        dir.setLockFactory(new NoLockFactory());
+        startFiles = dir.listAll();
+        new IndexWriter(dir, new WhitespaceAnalyzer(), IndexWriter.MaxFieldLength.UNLIMITED).close();
+      } finally {
+        dir.setLockFactory(lf);
+      }
       String[] endFiles = dir.listAll();
 
       Arrays.sort(startFiles);
@@ -3309,7 +3316,7 @@ public class TestIndexWriter extends Luc
   // LUCENE-510
   public void testAllUnicodeChars() throws Throwable {
 
-    UnicodeUtil.UTF8Result utf8 = new UnicodeUtil.UTF8Result();
+    BytesRef utf8 = new BytesRef(10);
     UnicodeUtil.UTF16Result utf16 = new UnicodeUtil.UTF16Result();
     char[] chars = new char[2];
     for(int ch=0;ch<0x0010FFFF;ch++) {
@@ -3329,16 +3336,16 @@ public class TestIndexWriter extends Luc
       UnicodeUtil.UTF16toUTF8(chars, 0, len, utf8);
       
       String s1 = new String(chars, 0, len);
-      String s2 = new String(utf8.result, 0, utf8.length, "UTF-8");
+      String s2 = new String(utf8.bytes, 0, utf8.length, "UTF-8");
       assertEquals("codepoint " + ch, s1, s2);
 
-      UnicodeUtil.UTF8toUTF16(utf8.result, 0, utf8.length, utf16);
+      UnicodeUtil.UTF8toUTF16(utf8.bytes, 0, utf8.length, utf16);
       assertEquals("codepoint " + ch, s1, new String(utf16.result, 0, utf16.length));
 
       byte[] b = s1.getBytes("UTF-8");
       assertEquals(utf8.length, b.length);
       for(int j=0;j<utf8.length;j++)
-        assertEquals(utf8.result[j], b[j]);
+        assertEquals(utf8.bytes[j], b[j]);
     }
   }
 
@@ -3403,7 +3410,7 @@ public class TestIndexWriter extends Luc
     char[] buffer = new char[20];
     char[] expected = new char[20];
 
-    UnicodeUtil.UTF8Result utf8 = new UnicodeUtil.UTF8Result();
+    BytesRef utf8 = new BytesRef(20);
     UnicodeUtil.UTF16Result utf16 = new UnicodeUtil.UTF16Result();
 
     for(int iter=0;iter<100000;iter++) {
@@ -3414,10 +3421,10 @@ public class TestIndexWriter extends Luc
         byte[] b = new String(buffer, 0, 20).getBytes("UTF-8");
         assertEquals(b.length, utf8.length);
         for(int i=0;i<b.length;i++)
-          assertEquals(b[i], utf8.result[i]);
+          assertEquals(b[i], utf8.bytes[i]);
       }
 
-      UnicodeUtil.UTF8toUTF16(utf8.result, 0, utf8.length, utf16);
+      UnicodeUtil.UTF8toUTF16(utf8.bytes, 0, utf8.length, utf16);
       assertEquals(utf16.length, 20);
       for(int i=0;i<20;i++)
         assertEquals(expected[i], utf16.result[i]);
@@ -3430,7 +3437,7 @@ public class TestIndexWriter extends Luc
     char[] buffer = new char[20];
     char[] expected = new char[20];
 
-    UnicodeUtil.UTF8Result utf8 = new UnicodeUtil.UTF8Result();
+    BytesRef utf8 = new BytesRef(20);
     UnicodeUtil.UTF16Result utf16 = new UnicodeUtil.UTF16Result();
     UnicodeUtil.UTF16Result utf16a = new UnicodeUtil.UTF16Result();
 
@@ -3453,7 +3460,7 @@ public class TestIndexWriter extends Luc
         byte[] b = new String(buffer, 0, 20).getBytes("UTF-8");
         assertEquals(b.length, utf8.length);
         for(int i=0;i<b.length;i++)
-          assertEquals(b[i], utf8.result[i]);
+          assertEquals(b[i], utf8.bytes[i]);
       }
 
       int bytePrefix = 20;
@@ -3461,18 +3468,18 @@ public class TestIndexWriter extends Luc
         bytePrefix = 0;
       else
         for(int i=0;i<20;i++)
-          if (last[i] != utf8.result[i]) {
+          if (last[i] != utf8.bytes[i]) {
             bytePrefix = i;
             break;
           }
-      System.arraycopy(utf8.result, 0, last, 0, utf8.length);
+      System.arraycopy(utf8.bytes, 0, last, 0, utf8.length);
 
-      UnicodeUtil.UTF8toUTF16(utf8.result, bytePrefix, utf8.length-bytePrefix, utf16);
+      UnicodeUtil.UTF8toUTF16(utf8.bytes, bytePrefix, utf8.length-bytePrefix, utf16);
       assertEquals(20, utf16.length);
       for(int i=0;i<20;i++)
         assertEquals(expected[i], utf16.result[i]);
 
-      UnicodeUtil.UTF8toUTF16(utf8.result, 0, utf8.length, utf16a);
+      UnicodeUtil.UTF8toUTF16(utf8.bytes, 0, utf8.length, utf16a);
       assertEquals(20, utf16a.length);
       for(int i=0;i<20;i++)
         assertEquals(expected[i], utf16a.result[i]);
@@ -4331,10 +4338,6 @@ public class TestIndexWriter extends Luc
 
       assertTrue(dir.fileExists("myrandomfile"));
 
-      // Make sure this does not copy myrandomfile:
-      Directory dir2 = new RAMDirectory(dir);
-      assertTrue(!dir2.fileExists("myrandomfile"));
-
     } finally {
       dir.close();
       _TestUtil.rmDir(indexDir);

Modified: lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/index/TestIndexWriterDelete.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/index/TestIndexWriterDelete.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/index/TestIndexWriterDelete.java (original)
+++ lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/index/TestIndexWriterDelete.java Tue Apr  6 19:19:27 2010
@@ -784,20 +784,8 @@ public class TestIndexWriterDelete exten
       }
     }
 
-    String[] startFiles = dir.listAll();
-    SegmentInfos infos = new SegmentInfos();
-    infos.read(dir);
-    new IndexFileDeleter(dir, new KeepOnlyLastCommitDeletionPolicy(), infos, null, null);
-    String[] endFiles = dir.listAll();
-
-    if (!Arrays.equals(startFiles, endFiles)) {
-      fail("docswriter abort() failed to delete unreferenced files:\n  before delete:\n    "
-           + arrayToString(startFiles) + "\n  after delete:\n    "
-           + arrayToString(endFiles));
-    }
-
-    modifier.close();
-
+      TestIndexWriter.assertNoUnreferencedFiles(dir, "docsWriter.abort() failed to delete unreferenced files");
+      modifier.close();
   }
 
   private String arrayToString(String[] l) {

Modified: lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/index/TestIndexWriterReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/index/TestIndexWriterReader.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/index/TestIndexWriterReader.java (original)
+++ lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/index/TestIndexWriterReader.java Tue Apr  6 19:19:27 2010
@@ -86,7 +86,7 @@ public class TestIndexWriterReader exten
 
     // get a reader
     IndexReader r1 = writer.getReader();
-    assertTrue(r1.isCurrent());
+    //assertTrue(r1.isCurrent());
 
     String id10 = r1.document(10).getField("id").stringValue();
     
@@ -94,7 +94,7 @@ public class TestIndexWriterReader exten
     newDoc.removeField("id");
     newDoc.add(new Field("id", Integer.toString(8000), Store.YES, Index.NOT_ANALYZED));
     writer.updateDocument(new Term("id", id10), newDoc);
-    assertFalse(r1.isCurrent());
+    //assertFalse(r1.isCurrent());
 
     IndexReader r2 = writer.getReader();
     assertTrue(r2.isCurrent());
@@ -157,7 +157,7 @@ public class TestIndexWriterReader exten
     IndexReader r0 = writer.getReader();
     assertTrue(r0.isCurrent());
     writer.addIndexesNoOptimize(new Directory[] { dir2 });
-    assertFalse(r0.isCurrent());
+    //assertFalse(r0.isCurrent());
     r0.close();
 
     IndexReader r1 = writer.getReader();

Modified: lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/index/TestLazyProxSkipping.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/index/TestLazyProxSkipping.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/index/TestLazyProxSkipping.java (original)
+++ lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/index/TestLazyProxSkipping.java Tue Apr  6 19:19:27 2010
@@ -48,7 +48,7 @@ public class TestLazyProxSkipping extend
       @Override
       public IndexInput openInput(String name) throws IOException {
         IndexInput ii = super.openInput(name);
-        if (name.endsWith(".prx")) {
+        if (name.endsWith(".prx") || name.endsWith(".pos")) {
           // we decorate the proxStream with a wrapper class that allows to count the number of calls of seek()
           ii = new SeeksCountingStream(ii);
         }

Modified: lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/index/TestMultiLevelSkipList.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/index/TestMultiLevelSkipList.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/index/TestMultiLevelSkipList.java (original)
+++ lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/index/TestMultiLevelSkipList.java Tue Apr  6 19:19:27 2010
@@ -30,6 +30,7 @@ import org.apache.lucene.document.Field;
 import org.apache.lucene.document.Field.Index;
 import org.apache.lucene.document.Field.Store;
 import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.MockRAMDirectory;
 import org.apache.lucene.store.RAMDirectory;
 import org.apache.lucene.util.LuceneTestCase;
 
@@ -42,6 +43,16 @@ import org.apache.lucene.util.LuceneTest
  * 
  */
 public class TestMultiLevelSkipList extends LuceneTestCase {
+
+  class CountingRAMDirectory extends MockRAMDirectory {
+    public IndexInput openInput(String fileName) throws IOException {
+      IndexInput in = super.openInput(fileName);
+      if (fileName.endsWith(".frq"))
+        in = new CountingStream(in);
+      return in;
+    }
+  }
+
   public void testSimpleSkip() throws IOException {
     RAMDirectory dir = new RAMDirectory();
     IndexWriter writer = new IndexWriter(dir, new PayloadAnalyzer(), true,
@@ -57,8 +68,7 @@ public class TestMultiLevelSkipList exte
     writer.close();
 
     IndexReader reader = SegmentReader.getOnlySegmentReader(dir);
-    SegmentTermPositions tp = (SegmentTermPositions) reader.termPositions();
-    tp.freqStream = new CountingStream(tp.freqStream);
+    TermPositions tp = reader.termPositions();
 
     for (int i = 0; i < 2; i++) {
       counter = 0;

Modified: lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/index/TestPayloads.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/index/TestPayloads.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/index/TestPayloads.java (original)
+++ lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/index/TestPayloads.java Tue Apr  6 19:19:27 2010
@@ -39,6 +39,7 @@ import org.apache.lucene.document.Field;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.FSDirectory;
 import org.apache.lucene.store.RAMDirectory;
+import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.util.UnicodeUtil;
 import org.apache.lucene.util._TestUtil;
@@ -257,10 +258,12 @@ public class TestPayloads extends Lucene
         tp.next();
         tp.nextPosition();
         // now we don't read this payload
+        tp.next();
         tp.nextPosition();
         assertEquals("Wrong payload length.", 1, tp.getPayloadLength());
         byte[] payload = tp.getPayload(null, 0);
         assertEquals(payload[0], payloadData[numTerms]);
+        tp.next();
         tp.nextPosition();
         
         // we don't read this payload and skip to a different document
@@ -559,13 +562,13 @@ public class TestPayloads extends Lucene
             }
         }
         
-        private UnicodeUtil.UTF8Result utf8Result = new UnicodeUtil.UTF8Result();
+        private BytesRef utf8Result = new BytesRef(10);
 
         synchronized String bytesToString(byte[] bytes) {
             String s = new String(bytes);
             UnicodeUtil.UTF16toUTF8(s, 0, s.length(), utf8Result);
             try {
-                return new String(utf8Result.result, 0, utf8Result.length, "UTF-8");
+                return new String(utf8Result.bytes, 0, utf8Result.length, "UTF-8");
             } catch (UnsupportedEncodingException uee) {
                 return null;
             }

Modified: lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/index/TestSegmentMerger.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/index/TestSegmentMerger.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/index/TestSegmentMerger.java (original)
+++ lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/index/TestSegmentMerger.java Tue Apr  6 19:19:27 2010
@@ -18,9 +18,11 @@ package org.apache.lucene.index;
  */
 
 import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.store.BufferedIndexInput;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.RAMDirectory;
 import org.apache.lucene.document.Document;
+import org.apache.lucene.index.codecs.CodecProvider;
 
 import java.io.IOException;
 import java.util.Collection;
@@ -63,14 +65,16 @@ public class TestSegmentMerger extends L
   }
   
   public void testMerge() throws IOException {                             
-    SegmentMerger merger = new SegmentMerger(mergedDir, mergedSegment);
+    SegmentMerger merger = new SegmentMerger(mergedDir, IndexWriter.DEFAULT_TERM_INDEX_INTERVAL, mergedSegment, null, CodecProvider.getDefault());
     merger.add(reader1);
     merger.add(reader2);
     int docsMerged = merger.merge();
     merger.closeReaders();
     assertTrue(docsMerged == 2);
     //Should be able to open a new SegmentReader against the new directory
-    SegmentReader mergedReader = SegmentReader.get(true, new SegmentInfo(mergedSegment, docsMerged, mergedDir, false, true), IndexReader.DEFAULT_TERMS_INDEX_DIVISOR);
+    SegmentReader mergedReader = SegmentReader.get(false, mergedDir, new SegmentInfo(mergedSegment, docsMerged, mergedDir, false, true,
+        -1, null, false, merger.hasProx(), merger.getCodec()), BufferedIndexInput.BUFFER_SIZE, true, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR, null);
+
     assertTrue(mergedReader != null);
     assertTrue(mergedReader.numDocs() == 2);
     Document newDoc1 = mergedReader.document(0);

Modified: lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/index/TestSegmentReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/index/TestSegmentReader.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/index/TestSegmentReader.java (original)
+++ lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/index/TestSegmentReader.java Tue Apr  6 19:19:27 2010
@@ -137,6 +137,7 @@ public class TestSegmentReader extends L
     TermPositions positions = reader.termPositions();
     positions.seek(new Term(DocHelper.TEXT_FIELD_1_KEY, "field"));
     assertTrue(positions != null);
+    assertTrue(positions.next());
     assertTrue(positions.doc() == 0);
     assertTrue(positions.nextPosition() >= 0);
   }    

Modified: lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/index/TestSegmentTermDocs.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/index/TestSegmentTermDocs.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/index/TestSegmentTermDocs.java (original)
+++ lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/index/TestSegmentTermDocs.java Tue Apr  6 19:19:27 2010
@@ -56,14 +56,13 @@ public class TestSegmentTermDocs extends
     SegmentReader reader = SegmentReader.get(true, info, indexDivisor);
     assertTrue(reader != null);
     assertEquals(indexDivisor, reader.getTermInfosIndexDivisor());
-    SegmentTermDocs segTermDocs = new SegmentTermDocs(reader);
-    assertTrue(segTermDocs != null);
-    segTermDocs.seek(new Term(DocHelper.TEXT_FIELD_2_KEY, "field"));
-    if (segTermDocs.next() == true)
-    {
-      int docId = segTermDocs.doc();
+    TermDocs termDocs = reader.termDocs();
+    assertTrue(termDocs != null);
+    termDocs.seek(new Term(DocHelper.TEXT_FIELD_2_KEY, "field"));
+    if (termDocs.next() == true) {
+      int docId = termDocs.doc();
       assertTrue(docId == 0);
-      int freq = segTermDocs.freq();
+      int freq = termDocs.freq();
       assertTrue(freq == 3);  
     }
     reader.close();
@@ -78,20 +77,20 @@ public class TestSegmentTermDocs extends
       //After adding the document, we should be able to read it back in
       SegmentReader reader = SegmentReader.get(true, info, indexDivisor);
       assertTrue(reader != null);
-      SegmentTermDocs segTermDocs = new SegmentTermDocs(reader);
-      assertTrue(segTermDocs != null);
-      segTermDocs.seek(new Term("textField2", "bad"));
-      assertTrue(segTermDocs.next() == false);
+      TermDocs termDocs = reader.termDocs();
+      assertTrue(termDocs != null);
+      termDocs.seek(new Term("textField2", "bad"));
+      assertTrue(termDocs.next() == false);
       reader.close();
     }
     {
       //After adding the document, we should be able to read it back in
       SegmentReader reader = SegmentReader.get(true, info, indexDivisor);
       assertTrue(reader != null);
-      SegmentTermDocs segTermDocs = new SegmentTermDocs(reader);
-      assertTrue(segTermDocs != null);
-      segTermDocs.seek(new Term("junk", "bad"));
-      assertTrue(segTermDocs.next() == false);
+      TermDocs termDocs = reader.termDocs();
+      assertTrue(termDocs != null);
+      termDocs.seek(new Term("junk", "bad"));
+      assertTrue(termDocs.next() == false);
       reader.close();
     }
   }

Modified: lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/index/TestSegmentTermEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/index/TestSegmentTermEnum.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/index/TestSegmentTermEnum.java (original)
+++ lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/index/TestSegmentTermEnum.java Tue Apr  6 19:19:27 2010
@@ -61,23 +61,6 @@ public class TestSegmentTermEnum extends
     verifyDocFreq();
   }
 
-  public void testPrevTermAtEnd() throws IOException
-  {
-    Directory dir = new MockRAMDirectory();
-    IndexWriter writer  = new IndexWriter(dir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED);
-    addDoc(writer, "aaa bbb");
-    writer.close();
-    SegmentReader reader = SegmentReader.getOnlySegmentReader(dir);
-    SegmentTermEnum termEnum = (SegmentTermEnum) reader.terms();
-    assertTrue(termEnum.next());
-    assertEquals("aaa", termEnum.term().text());
-    assertTrue(termEnum.next());
-    assertEquals("aaa", termEnum.prev().text());
-    assertEquals("bbb", termEnum.term().text());
-    assertFalse(termEnum.next());
-    assertEquals("bbb", termEnum.prev().text());
-  }
-
   private void verifyDocFreq()
       throws IOException
   {

Modified: lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/index/TestStressIndexing2.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/index/TestStressIndexing2.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/index/TestStressIndexing2.java (original)
+++ lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/index/TestStressIndexing2.java Tue Apr  6 19:19:27 2010
@@ -352,7 +352,7 @@ public class TestStressIndexing2 extends
         if (!termEnum1.next()) break;
       }
 
-       // iterate until we get some docs
+      // iterate until we get some docs
       int len2;
       for(;;) {
         len2=0;
@@ -369,12 +369,12 @@ public class TestStressIndexing2 extends
         if (!termEnum2.next()) break;
       }
 
-      if (!hasDeletes)
-        assertEquals(termEnum1.docFreq(), termEnum2.docFreq());
-
       assertEquals(len1, len2);
       if (len1==0) break;  // no more terms
 
+      if (!hasDeletes)
+        assertEquals(termEnum1.docFreq(), termEnum2.docFreq());
+
       assertEquals(term1, term2);
 
       // sort info2 to get it into ascending docid

Modified: lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/search/CheckHits.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/search/CheckHits.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/search/CheckHits.java (original)
+++ lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/search/CheckHits.java Tue Apr  6 19:19:27 2010
@@ -33,7 +33,7 @@ public class CheckHits {
    * different  order of operations from the actual scoring method ...
    * this allows for a small amount of variation
    */
-  public static float EXPLAIN_SCORE_TOLERANCE_DELTA = 0.00005f;
+  public static float EXPLAIN_SCORE_TOLERANCE_DELTA = 0.0002f;
     
   /**
    * Tests that all documents up to maxDoc which are *not* in the

Modified: lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/search/TestCachingWrapperFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/search/TestCachingWrapperFilter.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/search/TestCachingWrapperFilter.java (original)
+++ lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/search/TestCachingWrapperFilter.java Tue Apr  6 19:19:27 2010
@@ -65,7 +65,7 @@ public class TestCachingWrapperFilter ex
     if (originalSet.isCacheable()) {
       assertEquals("Cached DocIdSet must be of same class like uncached, if cacheable", originalSet.getClass(), cachedSet.getClass());
     } else {
-      assertTrue("Cached DocIdSet must be an OpenBitSet if the original one was not cacheable", cachedSet instanceof OpenBitSetDISI);
+      assertTrue("Cached DocIdSet must be an OpenBitSet if the original one was not cacheable", cachedSet instanceof OpenBitSetDISI || cachedSet == DocIdSet.EMPTY_DOCIDSET);
     }
   }
   

Modified: lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/search/TestNumericRangeQuery32.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/search/TestNumericRangeQuery32.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/search/TestNumericRangeQuery32.java (original)
+++ lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/search/TestNumericRangeQuery32.java Tue Apr  6 19:19:27 2010
@@ -230,6 +230,8 @@ public class TestNumericRangeQuery32 ext
     testRightOpenRange(2);
   }
   
+  /* TESTs disabled, because incompatible API change in 3.1/flex:
+  
   private void testRandomTrieAndClassicRangeQuery(int precisionStep) throws Exception {
     final Random rnd=newRandom();
     String field="field"+precisionStep;
@@ -298,6 +300,8 @@ public class TestNumericRangeQuery32 ext
     testRandomTrieAndClassicRangeQuery(Integer.MAX_VALUE);
   }
   
+  */
+  
   private void testRangeSplit(int precisionStep) throws Exception {
     final Random rnd=newRandom();
     String field="ascfield"+precisionStep;
@@ -443,37 +447,39 @@ public class TestNumericRangeQuery32 ext
     assertFalse(q2.equals(q1));
   }
   
-  private void testEnum(int lower, int upper) throws Exception {
-    NumericRangeQuery<Integer> q = NumericRangeQuery.newIntRange("field4", 4, lower, upper, true, true);
-    FilteredTermEnum termEnum = q.getEnum(searcher.getIndexReader());
-    try {
-      int count = 0;
-      do {
-        final Term t = termEnum.term();
-        if (t != null) {
-          final int val = NumericUtils.prefixCodedToInt(t.text());
-          assertTrue("value not in bounds", val >= lower && val <= upper);
-          count++;
-        } else break;
-      } while (termEnum.next());
-      assertFalse(termEnum.next());
-      System.out.println("TermEnum on 'field4' for range [" + lower + "," + upper + "] contained " + count + " terms.");
-    } finally {
-      termEnum.close();
-    }
-  }
+//  Removed for now  - NumericRangeQuery does not currently implement getEnum 
   
-  public void testEnum() throws Exception {
-    int count=3000;
-    int lower=(distance*3/2)+startOffset, upper=lower + count*distance + (distance/3);
-    // test enum with values
-    testEnum(lower, upper);
-    // test empty enum
-    testEnum(upper, lower);
-    // test empty enum outside of bounds
-    lower = distance*noDocs+startOffset;
-    upper = 2 * lower;
-    testEnum(lower, upper);
-  }
+//  private void testEnum(int lower, int upper) throws Exception {
+//    NumericRangeQuery<Integer> q = NumericRangeQuery.newIntRange("field4", 4, lower, upper, true, true);
+//    FilteredTermEnum termEnum = q.getEnum(searcher.getIndexReader());
+//    try {
+//      int count = 0;
+//      do {
+//        final Term t = termEnum.term();
+//        if (t != null) {
+//          final int val = NumericUtils.prefixCodedToInt(t.text());
+//          assertTrue("value not in bounds", val >= lower && val <= upper);
+//          count++;
+//        } else break;
+//      } while (termEnum.next());
+//      assertFalse(termEnum.next());
+//      System.out.println("TermEnum on 'field4' for range [" + lower + "," + upper + "] contained " + count + " terms.");
+//    } finally {
+//      termEnum.close();
+//    }
+//  }
+//  
+//  public void testEnum() throws Exception {
+//    int count=3000;
+//    int lower=(distance*3/2)+startOffset, upper=lower + count*distance + (distance/3);
+//    // test enum with values
+//    testEnum(lower, upper);
+//    // test empty enum
+//    testEnum(upper, lower);
+//    // test empty enum outside of bounds
+//    lower = distance*noDocs+startOffset;
+//    upper = 2 * lower;
+//    testEnum(lower, upper);
+//  }
   
 }

Modified: lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/search/TestNumericRangeQuery64.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/search/TestNumericRangeQuery64.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/search/TestNumericRangeQuery64.java (original)
+++ lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/search/TestNumericRangeQuery64.java Tue Apr  6 19:19:27 2010
@@ -245,6 +245,8 @@ public class TestNumericRangeQuery64 ext
     testRightOpenRange(2);
   }
   
+  /* TESTs disabled, because incompatible API change in 3.1/flex:
+  
   private void testRandomTrieAndClassicRangeQuery(int precisionStep) throws Exception {
     final Random rnd=newRandom();
     String field="field"+precisionStep;
@@ -317,6 +319,8 @@ public class TestNumericRangeQuery64 ext
     testRandomTrieAndClassicRangeQuery(Integer.MAX_VALUE);
   }
   
+  */
+  
   private void testRangeSplit(int precisionStep) throws Exception {
     final Random rnd=newRandom();
     String field="ascfield"+precisionStep;

Modified: lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/search/TestSort.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/search/TestSort.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/search/TestSort.java (original)
+++ lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/search/TestSort.java Tue Apr  6 19:19:27 2010
@@ -35,6 +35,7 @@ import org.apache.lucene.index.CorruptIn
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.Term;
+import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.queryParser.ParseException;
 import org.apache.lucene.search.BooleanClause.Occur;
 import org.apache.lucene.store.LockObtainFailedException;
@@ -332,20 +333,28 @@ public class TestSort extends LuceneTest
     FieldCache fc = FieldCache.DEFAULT;
 
 
-    sort.setSort (new SortField ("parser", new FieldCache.IntParser(){
-      public final int parseInt(final String val) {
-        return (val.charAt(0)-'A') * 123456;
+    sort.setSort ( new SortField ("parser", new FieldCache.IntParser(){
+      public final int parseInt(final String term) {
+        // dummy
+        return 0;
       }
-    }), SortField.FIELD_DOC );
+      public final int parseInt(final BytesRef term) {
+        return (term.bytes[term.offset]-'A') * 123456;
+      }
+    }), SortField.FIELD_DOC);
     assertMatches (full, queryA, sort, "JIHGFEDCBA");
     assertSaneFieldCaches(getName() + " IntParser");
     fc.purgeAllCaches();
 
-    sort.setSort (new SortField ("parser", new FieldCache.FloatParser(){
-      public final float parseFloat(final String val) {
-        return (float) Math.sqrt( val.charAt(0) );
+    sort.setSort (new SortField[] { new SortField ("parser", new FieldCache.FloatParser(){
+      public final float parseFloat(final String term) {
+        // dummy
+        return 0;
+      }
+      public final float parseFloat(final BytesRef term) {
+        return (float) Math.sqrt( term.bytes[term.offset] );
       }
-    }), SortField.FIELD_DOC );
+    }), SortField.FIELD_DOC });
     assertMatches (full, queryA, sort, "JIHGFEDCBA");
     assertSaneFieldCaches(getName() + " FloatParser");
     fc.purgeAllCaches();
@@ -354,34 +363,49 @@ public class TestSort extends LuceneTest
       public final long parseLong(final String val) {
         return (val.charAt(0)-'A') * 1234567890L;
       }
-    }), SortField.FIELD_DOC );
+      public final long parseLong(final BytesRef term) {
+        return (term.bytes[term.offset]-'A') * 1234567890L;
+      }
+    }), SortField.FIELD_DOC);
     assertMatches (full, queryA, sort, "JIHGFEDCBA");
     assertSaneFieldCaches(getName() + " LongParser");
     fc.purgeAllCaches();
 
-    sort.setSort (new SortField ("parser", new FieldCache.DoubleParser(){
-      public final double parseDouble(final String val) {
-        return Math.pow( val.charAt(0), (val.charAt(0)-'A') );
+    sort.setSort (new SortField[] { new SortField ("parser", new FieldCache.DoubleParser(){
+      public final double parseDouble(final String term) {
+        // dummy
+        return 0;
+      }
+      public final double parseDouble(final BytesRef term) {
+        return Math.pow( term.bytes[term.offset], (term.bytes[term.offset]-'A') );
       }
-    }), SortField.FIELD_DOC );
+    }), SortField.FIELD_DOC });
     assertMatches (full, queryA, sort, "JIHGFEDCBA");
     assertSaneFieldCaches(getName() + " DoubleParser");
     fc.purgeAllCaches();
 
-    sort.setSort (new SortField ("parser", new FieldCache.ByteParser(){
-      public final byte parseByte(final String val) {
-        return (byte) (val.charAt(0)-'A');
+    sort.setSort (new SortField[] { new SortField ("parser", new FieldCache.ByteParser(){
+      public final byte parseByte(final String term) {
+        // dummy
+        return 0;
       }
-    }), SortField.FIELD_DOC );
+      public final byte parseByte(final BytesRef term) {
+        return (byte) (term.bytes[term.offset]-'A');
+      }
+    }), SortField.FIELD_DOC });
     assertMatches (full, queryA, sort, "JIHGFEDCBA");
     assertSaneFieldCaches(getName() + " ByteParser");
     fc.purgeAllCaches();
 
-    sort.setSort (new SortField ("parser", new FieldCache.ShortParser(){
-      public final short parseShort(final String val) {
-        return (short) (val.charAt(0)-'A');
+    sort.setSort (new SortField[] { new SortField ("parser", new FieldCache.ShortParser(){
+      public final short parseShort(final String term) {
+        // dummy
+        return 0;
+      }
+      public final short parseShort(final BytesRef term) {
+        return (short) (term.bytes[term.offset]-'A');
       }
-    }), SortField.FIELD_DOC );
+    }), SortField.FIELD_DOC });
     assertMatches (full, queryA, sort, "JIHGFEDCBA");
     assertSaneFieldCaches(getName() + " ShortParser");
     fc.purgeAllCaches();
@@ -439,8 +463,12 @@ public class TestSort extends LuceneTest
     @Override
     public void setNextReader(IndexReader reader, int docBase) throws IOException {
       docValues = FieldCache.DEFAULT.getInts(reader, "parser", new FieldCache.IntParser() {
-          public final int parseInt(final String val) {
-            return (val.charAt(0)-'A') * 123456;
+          public final int parseInt(final String term) {
+            // dummy
+            return 0;
+          }
+          public final int parseInt(final BytesRef term) {
+            return (term.bytes[term.offset]-'A') * 123456;
           }
         });
     }

Modified: lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/search/TestTermScorer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/search/TestTermScorer.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/search/TestTermScorer.java (original)
+++ lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/search/TestTermScorer.java Tue Apr  6 19:19:27 2010
@@ -72,9 +72,9 @@ public class TestTermScorer extends Luce
 
         Weight weight = termQuery.weight(indexSearcher);
 
-        TermScorer ts = new TermScorer(weight,
-                                       indexReader.termDocs(allTerm), indexSearcher.getSimilarity(),
-                                       indexReader.norms(FIELD));
+        Scorer ts = weight.scorer(indexSearcher.getIndexReader(),
+                                  true, true);
+
         //we have 2 documents with the term all in them, one document for all the other values
         final List docs = new ArrayList();
         //must call next first
@@ -138,9 +138,9 @@ public class TestTermScorer extends Luce
 
         Weight weight = termQuery.weight(indexSearcher);
 
-        TermScorer ts = new TermScorer(weight,
-                                       indexReader.termDocs(allTerm), indexSearcher.getSimilarity(),
-                                       indexReader.norms(FIELD));
+        Scorer ts = weight.scorer(indexSearcher.getIndexReader(),
+                                  true, true);
+ 
         assertTrue("next did not return a doc", ts.nextDoc() != DocIdSetIterator.NO_MORE_DOCS);
         assertTrue("score is not correct", ts.score() == 1.6931472f);
         assertTrue("next did not return a doc", ts.nextDoc() != DocIdSetIterator.NO_MORE_DOCS);
@@ -155,9 +155,9 @@ public class TestTermScorer extends Luce
 
         Weight weight = termQuery.weight(indexSearcher);
 
-        TermScorer ts = new TermScorer(weight,
-                                       indexReader.termDocs(allTerm), indexSearcher.getSimilarity(),
-                                       indexReader.norms(FIELD));
+        Scorer ts = weight.scorer(indexSearcher.getIndexReader(),
+                                  true, true);
+
         assertTrue("Didn't skip", ts.advance(3) != DocIdSetIterator.NO_MORE_DOCS);
         //The next doc should be doc 5
         assertTrue("doc should be number 5", ts.docID() == 5);

Modified: lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/search/TestWildcard.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/search/TestWildcard.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/search/TestWildcard.java (original)
+++ lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/search/TestWildcard.java Tue Apr  6 19:19:27 2010
@@ -114,6 +114,7 @@ public class TestWildcard
    * rewritten to a single PrefixQuery. The boost and rewriteMethod should be
    * preserved.
    */
+  /* disable because rewrites changed in flex/trunk
   public void testPrefixTerm() throws IOException {
     RAMDirectory indexStore = getIndexStore("field", new String[]{"prefix", "prefixx"});
     IndexSearcher searcher = new IndexSearcher(indexStore, true);
@@ -145,7 +146,7 @@ public class TestWildcard
     expected.setRewriteMethod(wq.getRewriteMethod());
     expected.setBoost(wq.getBoost());
     assertEquals(searcher.rewrite(expected), searcher.rewrite(wq));
-  }
+  }*/
 
   /**
    * Tests Wildcard queries with an asterisk.

Modified: lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/util/TestAttributeSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/util/TestAttributeSource.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/util/TestAttributeSource.java (original)
+++ lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/util/TestAttributeSource.java Tue Apr  6 19:19:27 2010
@@ -78,22 +78,22 @@ public class TestAttributeSource extends
   
   public void testCloneAttributes() {
     final AttributeSource src = new AttributeSource();
-    final TermAttribute termAtt = src.addAttribute(TermAttribute.class);
+    final FlagsAttribute flagsAtt = src.addAttribute(FlagsAttribute.class);
     final TypeAttribute typeAtt = src.addAttribute(TypeAttribute.class);
-    termAtt.setTermBuffer("TestTerm");
+    flagsAtt.setFlags(1234);
     typeAtt.setType("TestType");
     
     final AttributeSource clone = src.cloneAttributes();
     final Iterator<Class<? extends Attribute>> it = clone.getAttributeClassesIterator();
-    assertEquals("TermAttribute must be the first attribute", TermAttribute.class, it.next());
+    assertEquals("FlagsAttribute must be the first attribute", FlagsAttribute.class, it.next());
     assertEquals("TypeAttribute must be the second attribute", TypeAttribute.class, it.next());
     assertFalse("No more attributes", it.hasNext());
     
-    final TermAttribute termAtt2 = clone.getAttribute(TermAttribute.class);
+    final FlagsAttribute flagsAtt2 = clone.getAttribute(FlagsAttribute.class);
     final TypeAttribute typeAtt2 = clone.getAttribute(TypeAttribute.class);
-    assertNotSame("TermAttribute of original and clone must be different instances", termAtt2, termAtt);
+    assertNotSame("FlagsAttribute of original and clone must be different instances", flagsAtt2, flagsAtt);
     assertNotSame("TypeAttribute of original and clone must be different instances", typeAtt2, typeAtt);
-    assertEquals("TermAttribute of original and clone must be equal", termAtt2, termAtt);
+    assertEquals("FlagsAttribute of original and clone must be equal", flagsAtt2, flagsAtt);
     assertEquals("TypeAttribute of original and clone must be equal", typeAtt2, typeAtt);
   }
   

Propchange: lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/util/TestAttributeSource.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Apr  6 19:19:27 2010
@@ -1,6 +1,8 @@
+/lucene/java/branches/flex_1458/backwards/src/test/org/apache/lucene/util/TestAttributeSource.java:824912-931101
 /lucene/java/branches/lucene_2_4/src/test/org/apache/lucene/util/TestAttributeSource.java:748824
 /lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/util/TestAttributeSource.java:817269-818600,825998,829134,829881,831036
 /lucene/java/branches/lucene_2_9_back_compat_tests/src/test/org/apache/lucene/util/TestAttributeSource.java:818601-821336
 /lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/util/TestAttributeSource.java:880754,880793,880823,881216,881317,881376,881473,881549,881820,882467,882890,883076,883080,912383
-/lucene/java/trunk/backwards/src/test/org/apache/lucene/util/TestAttributeSource.java:924483-925561
+/lucene/java/branches/lucene_3_0_back_compat_tests/src/test/org/apache/lucene/util/TestAttributeSource.java:880869-912395
+/lucene/java/trunk/backwards/src/test/org/apache/lucene/util/TestAttributeSource.java:924483-924731,924781,925176-925462
 /lucene/java/trunk/src/test/org/apache/lucene/util/TestAttributeSource.java:881213,881315,881466,881819,882374,882807,882888,882977,883074-883075,883079,885214,912319,912324,912328,912335,912372

Modified: lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/util/TestNumericUtils.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/util/TestNumericUtils.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/util/TestNumericUtils.java (original)
+++ lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/util/TestNumericUtils.java Tue Apr  6 19:19:27 2010
@@ -26,6 +26,8 @@ import java.util.Iterator;
 
 public class TestNumericUtils extends LuceneTestCase {
 
+  /* TESTs disabled, because incompatible API change in 3.1/flex:
+  
   public void testLongConversionAndOrdering() throws Exception {
     // generate a series of encoded longs, each numerical one bigger than the one before
     String last=null;
@@ -131,6 +133,8 @@ public class TestNumericUtils extends Lu
       }
     }
   }
+  
+  */
 
   public void testDoubles() throws Exception {
     double[] vals=new double[]{

Modified: lucene/dev/trunk/lucene/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/build.xml?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/build.xml (original)
+++ lucene/dev/trunk/lucene/build.xml Tue Apr  6 19:19:27 2010
@@ -104,24 +104,24 @@ The source distribution does not contain
 
   <target name="compile-backwards" depends="compile-core, jar-core, test-backwards-message"
   	description="Runs tests of a previous Lucene version." if="backwards.available">
-    <sequential>
+	<sequential>
       <mkdir dir="${build.dir.backwards}"/>	  
 
-      <!-- first compile branch classes -->
-      <compile
+	  <!-- first compile branch classes -->
+	  <compile
         srcdir="${backwards.dir}/src/java"
         destdir="${build.dir.backwards}/classes/java"
         javac.source="${javac.source.backwards}" javac.target="${javac.target.backwards}"
       >
         <classpath refid="backwards.compile.classpath"/>
-      </compile>
-          
+	  </compile>
+		  	
       <!-- compile branch tests against branch classpath -->	
       <compile-test-macro srcdir="${backwards.dir}/src/test" destdir="${build.dir.backwards}/classes/test"
                   test.classpath="backwards.test.compile.classpath" javac.source="${javac.source.backwards}" javac.target="${javac.target.backwards}"/>
       
 
-    </sequential>
+  	</sequential>
   </target>	
 
   <target name="test-backwards" depends="compile-backwards, junit-backwards-mkdir, junit-backwards-sequential, junit-backwards-parallel"/>
@@ -715,6 +715,41 @@ The source distribution does not contain
     </delete>
   </target>
 
+  <macrodef name="createLevAutomaton">
+  	<attribute name="n"/>
+  	<sequential>
+  		<exec dir="src/java/org/apache/lucene/util/automaton"
+  			executable="${python.exe}" failonerror="true">
+  	  	<arg line="createLevAutomata.py @{n}"/>
+  	</exec>
+  	</sequential>
+  </macrodef>
+
+  <target name="createLevAutomata" depends="check-moman,clone-moman,pull-moman">
+  	<createLevAutomaton n="1"/>
+  	<createLevAutomaton n="2"/>
+  </target>
+  
+  <target name="check-moman">
+  	<condition property="moman.cloned">
+      <available file="src/java/org/apache/lucene/util/automaton/moman"/>
+  	</condition>
+  </target>
+	
+  <target name="clone-moman" unless="moman.cloned">
+  	<exec dir="src/java/org/apache/lucene/util/automaton" 
+  		executable="${hg.exe}" failonerror="true">
+      <arg line="clone -r ${moman.rev} ${moman.url} moman"/>
+    </exec>
+  </target>
+
+  <target name="pull-moman" if="moman.cloned">
+	<exec dir="src/java/org/apache/lucene/util/automaton/moman" 
+		executable="${hg.exe}" failonerror="true">
+	  <arg line="pull -f -r ${moman.rev}"/>
+	</exec>
+  </target>
+
   <macrodef name="contrib-crawl">
     <attribute name="target" default=""/>
     <attribute name="failonerror" default="true"/>

Propchange: lucene/dev/trunk/lucene/build.xml
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Apr  6 19:19:27 2010
@@ -1,2 +1,3 @@
-/lucene/java/branches/lucene_2_9/build.xml:896850,909334
-/lucene/java/trunk/build.xml:924483-925561
+/lucene/java/branches/flex_1458/build.xml:824912-931101
+/lucene/java/branches/lucene_2_9/build.xml:909334
+/lucene/java/trunk/build.xml:924483-924731,924781,925176-925462

Modified: lucene/dev/trunk/lucene/common-build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/common-build.xml?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/common-build.xml (original)
+++ lucene/dev/trunk/lucene/common-build.xml Tue Apr  6 19:19:27 2010
@@ -119,6 +119,11 @@
   <property name="svnversion.exe" value="svnversion" />
   <property name="svn.exe" value="svn" />
   
+  <property name="hg.exe" value="hg" />
+  <property name="moman.url" value="https://bitbucket.org/jpbarrette/moman" />
+  <property name="moman.rev" value="115" />
+  <property name="python.exe" value="python" />
+
   <property name="gpg.exe" value="gpg" />
   <property name="gpg.key" value="CODE SIGNING KEY" />
 

Propchange: lucene/dev/trunk/lucene/contrib/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Apr  6 19:19:27 2010
@@ -1,5 +1,6 @@
+/lucene/java/branches/flex_1458/contrib:824912-931101
 /lucene/java/branches/lucene_2_4/contrib:748824
 /lucene/java/branches/lucene_2_9/contrib:817269-818600,825998,829134,829816,829881,831036,896850,909334
 /lucene/java/branches/lucene_2_9_back_compat_tests/contrib:818601-821336
 /lucene/java/branches/lucene_3_0/contrib:880793,896906
-/lucene/java/trunk/contrib:924483-925561
+/lucene/java/trunk/contrib:924483-924731,924781,925176-925462

Propchange: lucene/dev/trunk/lucene/contrib/CHANGES.txt
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Apr  6 19:19:27 2010
@@ -1,5 +1,6 @@
+/lucene/java/branches/flex_1458/contrib/CHANGES.txt:824912-931101
 /lucene/java/branches/lucene_2_4/contrib/CHANGES.txt:748824
-/lucene/java/branches/lucene_2_9/contrib/CHANGES.txt:817269-818600,825998,826775,829134,829816,829881,831036,896850,909334
+/lucene/java/branches/lucene_2_9/contrib/CHANGES.txt:817269-818600,825998,826775,829134,829816,829881,831036,896850
 /lucene/java/branches/lucene_2_9_back_compat_tests/contrib/CHANGES.txt:818601-821336
 /lucene/java/branches/lucene_3_0/contrib/CHANGES.txt:880793,896906
-/lucene/java/trunk/contrib/CHANGES.txt:924483-925561
+/lucene/java/trunk/contrib/CHANGES.txt:924483-924731,924781,925176-925462

Modified: lucene/dev/trunk/lucene/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java (original)
+++ lucene/dev/trunk/lucene/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java Tue Apr  6 19:19:27 2010
@@ -37,11 +37,12 @@ import org.apache.lucene.benchmark.byTas
 import org.apache.lucene.collation.CollationKeyAnalyzer;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.index.MultiFields;
+import org.apache.lucene.index.FieldsEnum;
+import org.apache.lucene.index.DocsEnum;
 import org.apache.lucene.index.IndexWriterConfig;
 import org.apache.lucene.index.LogMergePolicy;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.index.TermEnum;
-import org.apache.lucene.index.TermDocs;
 import org.apache.lucene.index.SerialMergeScheduler;
 import org.apache.lucene.index.LogDocMergePolicy;
 import org.apache.lucene.index.TermFreqVector;
@@ -474,16 +475,20 @@ public class TestPerfTasksLogic extends 
     IndexReader reader = IndexReader.open(benchmark.getRunData().getDirectory(), true);
     assertEquals(NUM_DOCS, reader.numDocs());
 
-    TermEnum terms = reader.terms();
-    TermDocs termDocs = reader.termDocs();
     int totalTokenCount2 = 0;
-    while(terms.next()) {
-      Term term = terms.term();
-      /* not-tokenized, but indexed field */
-      if (term != null && term.field() != DocMaker.ID_FIELD) { 
-        termDocs.seek(terms.term());
-        while (termDocs.next())
-          totalTokenCount2 += termDocs.freq();
+
+    FieldsEnum fields = MultiFields.getFields(reader).iterator();
+    String fieldName = null;
+    while((fieldName = fields.next()) != null) {
+      if (fieldName == DocMaker.ID_FIELD)
+        continue;
+      TermsEnum terms = fields.terms();
+      DocsEnum docs = null;
+      while(terms.next() != null) {
+        docs = terms.docs(MultiFields.getDeletedDocs(reader), docs);
+        while(docs.nextDoc() != docs.NO_MORE_DOCS) {
+          totalTokenCount2 += docs.freq();
+        }
       }
     }
     reader.close();

Modified: lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTermExtractor.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTermExtractor.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTermExtractor.java (original)
+++ lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTermExtractor.java Tue Apr  6 19:19:27 2010
@@ -150,11 +150,16 @@ public class WeightedSpanTermExtractor {
         mtq.setRewriteMethod(MultiTermQuery.SCORING_BOOLEAN_QUERY_REWRITE);
         query = mtq;
       }
-      FakeReader fReader = new FakeReader();
-      MultiTermQuery.SCORING_BOOLEAN_QUERY_REWRITE.rewrite(fReader, mtq);
-      if (fReader.field != null) {
-        IndexReader ir = getReaderForField(fReader.field);
+      if (mtq.getField() != null) {
+        IndexReader ir = getReaderForField(mtq.getField());
         extract(query.rewrite(ir), terms);
+      } else {
+        FakeReader fReader = new FakeReader();
+        MultiTermQuery.SCORING_BOOLEAN_QUERY_REWRITE.rewrite(fReader, mtq);
+        if (fReader.field != null) {
+          IndexReader ir = getReaderForField(fReader.field);
+          extract(query.rewrite(ir), terms);
+        }
       }
     } else if (query instanceof MultiPhraseQuery) {
       final MultiPhraseQuery mpq = (MultiPhraseQuery) query;

Propchange: lucene/dev/trunk/lucene/contrib/highlighter/src/test/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Apr  6 19:19:27 2010
@@ -1,5 +1,6 @@
+/lucene/java/branches/flex_1458/contrib/highlighter/src/test:824912-931101
 /lucene/java/branches/lucene_2_4/contrib/highlighter/src/test:748824
 /lucene/java/branches/lucene_2_9/contrib/highlighter/src/test:817269-818600,825998,826775,829134,829816,829881,831036,896850,909334
 /lucene/java/branches/lucene_2_9_back_compat_tests/contrib/highlighter/src/test:818601-821336
 /lucene/java/branches/lucene_3_0/contrib/highlighter/src/test:880793,896906
-/lucene/java/trunk/contrib/highlighter/src/test:924483-925561
+/lucene/java/trunk/contrib/highlighter/src/test:924483-924731,924781,925176-925462

Propchange: lucene/dev/trunk/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Apr  6 19:19:27 2010
@@ -1,2 +1,3 @@
-/lucene/java/branches/lucene_2_9/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java:896850,909334
-/lucene/java/trunk/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java:924483-925561
+/lucene/java/branches/flex_1458/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java:824912-931101
+/lucene/java/branches/lucene_2_9/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java:909334
+/lucene/java/trunk/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java:924483-924731,924781,925176-925462

Modified: lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/index/FieldNormModifier.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/index/FieldNormModifier.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/index/FieldNormModifier.java (original)
+++ lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/index/FieldNormModifier.java Tue Apr  6 19:19:27 2010
@@ -19,11 +19,15 @@ package org.apache.lucene.index;
 import java.io.IOException;
 import java.io.File;
 import java.util.Date;
+import java.util.List;
+import java.util.ArrayList;
 
 import org.apache.lucene.search.Similarity;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.FSDirectory;
 import org.apache.lucene.util.StringHelper;
+import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.ReaderUtil;
 
 /**
  * Given a directory and a list of fields, updates the fieldNorms in place for every document.
@@ -104,46 +108,46 @@ public class FieldNormModifier {
    */
   public void reSetNorms(String field) throws IOException {
     String fieldName = StringHelper.intern(field);
-    int[] termCounts = new int[0];
     
     IndexReader reader = null;
-    TermEnum termEnum = null;
-    TermDocs termDocs = null;
     try {
-      reader = IndexReader.open(dir, true);
-      termCounts = new int[reader.maxDoc()];
-      try {
-        termEnum = reader.terms(new Term(field));
-        try {
-          termDocs = reader.termDocs();
-          do {
-            Term term = termEnum.term();
-            if (term != null && term.field().equals(fieldName)) {
-              termDocs.seek(termEnum.term());
-              while (termDocs.next()) {
-                termCounts[termDocs.doc()] += termDocs.freq();
+      reader = IndexReader.open(dir, false);
+
+      final List<IndexReader> subReaders = new ArrayList<IndexReader>();
+      ReaderUtil.gatherSubReaders(subReaders, reader);
+
+      for(IndexReader subReader : subReaders) {
+        final Bits delDocs = subReader.getDeletedDocs();
+
+        int[] termCounts = new int[subReader.maxDoc()];
+        Fields fields = subReader.fields();
+        if (fields != null) {
+          Terms terms = fields.terms(field);
+          if (terms != null) {
+            TermsEnum termsEnum = terms.iterator();
+            DocsEnum docs = null;
+            while(termsEnum.next() != null) {
+              docs = termsEnum.docs(delDocs, docs);
+              while(true) {
+                int docID = docs.nextDoc();
+                if (docID != docs.NO_MORE_DOCS) {
+                  termCounts[docID] += docs.freq();
+                } else {
+                  break;
+                }
               }
             }
-          } while (termEnum.next());
-          
-        } finally {
-          if (null != termDocs) termDocs.close();
+          }
         }
-      } finally {
-        if (null != termEnum) termEnum.close();
-      }
-    } finally {
-      if (null != reader) reader.close();
-    }
-    
-    try {
-      reader = IndexReader.open(dir, false); 
-      for (int d = 0; d < termCounts.length; d++) {
-        if (! reader.isDeleted(d)) {
-          if (sim == null)
-            reader.setNorm(d, fieldName, Similarity.encodeNorm(1.0f));
-          else
-            reader.setNorm(d, fieldName, sim.encodeNormValue(sim.lengthNorm(fieldName, termCounts[d])));
+
+        for (int d = 0; d < termCounts.length; d++) {
+          if (delDocs == null || !delDocs.get(d)) {
+            if (sim == null) {
+              subReader.setNorm(d, fieldName, Similarity.encodeNorm(1.0f));
+            } else {
+              subReader.setNorm(d, fieldName, sim.encodeNormValue(sim.lengthNorm(fieldName, termCounts[d])));
+            }
+          }
         }
       }
       
@@ -151,5 +155,4 @@ public class FieldNormModifier {
       if (null != reader) reader.close();
     }
   }
-  
 }

Modified: lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/index/MultiPassIndexSplitter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/index/MultiPassIndexSplitter.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/index/MultiPassIndexSplitter.java (original)
+++ lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/index/MultiPassIndexSplitter.java Tue Apr  6 19:19:27 2010
@@ -26,6 +26,7 @@ import org.apache.lucene.index.IndexWrit
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.FSDirectory;
 import org.apache.lucene.util.OpenBitSet;
+import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.Version;
 
 /**
@@ -172,6 +173,8 @@ public class MultiPassIndexSplitter {
    * list of deletions.
    */
   public static class FakeDeleteIndexReader extends FilterIndexReader {
+    // TODO: switch to flex api, here
+
     OpenBitSet dels;
     OpenBitSet oldDels = null;
 
@@ -202,6 +205,7 @@ public class MultiPassIndexSplitter {
       if (oldDels != null) {
         dels.or(oldDels);
       }
+      storeDelDocs(null);
     }
 
     @Override
@@ -215,6 +219,16 @@ public class MultiPassIndexSplitter {
     }
 
     @Override
+    public IndexReader[] getSequentialSubReaders() {
+      return null;
+    }
+
+    @Override
+    public Bits getDeletedDocs() {
+      return dels;
+    }
+
+    @Override
     public boolean isDeleted(int n) {
       return dels.get(n);
     }
@@ -235,5 +249,29 @@ public class MultiPassIndexSplitter {
         }        
       };
     }
+
+    @Override
+    public TermDocs termDocs() throws IOException {
+      return new FilterTermDocs(in.termDocs()) {
+
+        @Override
+        public boolean next() throws IOException {
+          boolean res;
+          while ((res = super.next())) {
+            if (!dels.get(doc())) {
+              break;
+            }
+          }
+          return res;
+        }        
+      };
+    }
+
+    @Override
+    public TermDocs termDocs(Term term) throws IOException {
+      TermDocs termDocs = termDocs();
+      termDocs.seek(term);
+      return termDocs;
+    }
   }
 }

Modified: lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/index/TermVectorAccessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/index/TermVectorAccessor.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/index/TermVectorAccessor.java (original)
+++ lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/index/TermVectorAccessor.java Tue Apr  6 19:19:27 2010
@@ -1,10 +1,5 @@
 package org.apache.lucene.index;
 
-import org.apache.lucene.util.StringHelper;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
 /*
  *  Licensed under the Apache License, Version 2.0 (the "License");
  *  you may not use this file except in compliance with the License.
@@ -20,6 +15,14 @@ import java.util.List;
  *
  */
 
+import org.apache.lucene.util.StringHelper;
+import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.BytesRef;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
 
 /**
  * Transparent access to the vector space model,
@@ -97,40 +100,53 @@ public class TermVectorAccessor {
       positions.clear();
     }
 
-    TermEnum termEnum = indexReader.terms(new Term(field, ""));
-    if (termEnum.term() != null) {
-      while (termEnum.term().field() == field) {
-        TermPositions termPositions = indexReader.termPositions(termEnum.term());
-        if (termPositions.skipTo(documentNumber)) {
-  
-          frequencies.add(Integer.valueOf(termPositions.freq()));
-          tokens.add(termEnum.term().text());
-  
-  
+    final Bits delDocs = MultiFields.getDeletedDocs(indexReader);
+
+    Terms terms = MultiFields.getTerms(indexReader, field);
+    boolean anyTerms = false;
+    if (terms != null) {
+      TermsEnum termsEnum = terms.iterator();
+      DocsEnum docs = null;
+      DocsAndPositionsEnum postings = null;
+      while(true) {
+        BytesRef text = termsEnum.next();
+        if (text != null) {
+          anyTerms = true;
           if (!mapper.isIgnoringPositions()) {
-            int[] positions = new int[termPositions.freq()];
-            for (int i = 0; i < positions.length; i++) {
-              positions[i] = termPositions.nextPosition();
-            }
-            this.positions.add(positions);
+            docs = postings = termsEnum.docsAndPositions(delDocs, postings);
           } else {
-            positions.add(null);
+            docs = termsEnum.docs(delDocs, docs);
           }
-        }
-        termPositions.close();
-        if (!termEnum.next()) {
+
+          int docID = docs.advance(documentNumber);
+          if (docID == documentNumber) {
+
+            frequencies.add(Integer.valueOf(docs.freq()));
+            tokens.add(text.utf8ToString());
+
+            if (!mapper.isIgnoringPositions()) {
+              int[] positions = new int[docs.freq()];
+              for (int i = 0; i < positions.length; i++) {
+                positions[i] = postings.nextPosition();
+              }
+              this.positions.add(positions);
+            } else {
+              positions.add(null);
+            }
+          }
+        } else {
           break;
         }
       }
-      mapper.setDocumentNumber(documentNumber);
-      mapper.setExpectations(field, tokens.size(), false, !mapper.isIgnoringPositions());
-      for (int i = 0; i < tokens.size(); i++) {
-        mapper.map(tokens.get(i), frequencies.get(i).intValue(), (TermVectorOffsetInfo[]) null, positions.get(i));
+
+      if (anyTerms) {
+        mapper.setDocumentNumber(documentNumber);
+        mapper.setExpectations(field, tokens.size(), false, !mapper.isIgnoringPositions());
+        for (int i = 0; i < tokens.size(); i++) {
+          mapper.map(tokens.get(i), frequencies.get(i).intValue(), (TermVectorOffsetInfo[]) null, positions.get(i));
+        }
       }
     }
-    termEnum.close();
-
-
   }
 
 

Modified: lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/misc/HighFreqTerms.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/misc/HighFreqTerms.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/misc/HighFreqTerms.java (original)
+++ lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/misc/HighFreqTerms.java Tue Apr  6 19:19:27 2010
@@ -18,7 +18,10 @@ package org.apache.lucene.misc;
 
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.Term;
-import org.apache.lucene.index.TermEnum;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.index.FieldsEnum;
+import org.apache.lucene.index.Terms;
 import org.apache.lucene.store.FSDirectory;
 import org.apache.lucene.util.PriorityQueue;
 
@@ -50,20 +53,40 @@ public class HighFreqTerms {
     }
 
     TermInfoQueue tiq = new TermInfoQueue(numTerms);
-    TermEnum terms = reader.terms();
 
     if (field != null) { 
-      while (terms.next()) {
-        if (terms.term().field().equals(field)) {
-          tiq.insertWithOverflow(new TermInfo(terms.term(), terms.docFreq()));
+      Terms terms = reader.fields().terms(field);
+      if (terms != null) {
+        TermsEnum termsEnum = terms.iterator();
+        while(true) {
+          BytesRef term = termsEnum.next();
+          if (term != null) {
+            tiq.insertWithOverflow(new TermInfo(new Term(field, term.utf8ToString()), termsEnum.docFreq()));
+          } else {
+            break;
+          }    
         }
       }
-    }
-    else {
-      while (terms.next()) {
-        tiq.insertWithOverflow(new TermInfo(terms.term(), terms.docFreq()));
+    } else {
+      FieldsEnum fields = reader.fields().iterator();
+      while(true) {
+        field = fields.next();
+        if (field != null) {
+          TermsEnum terms = fields.terms();
+          while(true) {
+            BytesRef term = terms.next();
+            if (term != null) {
+              tiq.insertWithOverflow(new TermInfo(new Term(field, term.toString()), terms.docFreq()));
+            } else {
+              break;
+            }
+          }
+        } else {
+          break;
+        }
       }
     }
+    
     while (tiq.size() != 0) {
       TermInfo termInfo = tiq.pop();
       System.out.println(termInfo.term + " " + termInfo.docFreq);



Mime
View raw message