lucene-java-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mikemcc...@apache.org
Subject svn commit: r920378 [3/3] - in /lucene/java/branches/flex_1458: contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/ contrib/misc/src/java/org/apache/lucene/index/ contrib/misc/src/test/org/apache/lucene/index/ contrib/queries/src/java/org/ap...
Date Mon, 08 Mar 2010 16:10:33 GMT
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/ReaderUtil.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/ReaderUtil.java?rev=920378&r1=920377&r2=920378&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/ReaderUtil.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/ReaderUtil.java Mon Mar
 8 16:10:31 2010
@@ -22,7 +22,6 @@
 import java.io.IOException;
 
 import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.Fields;
 
 /**
  * Common util methods for dealing with {@link IndexReader}s.
@@ -54,35 +53,53 @@
    * @param allSubReaders
    * @param reader
    */
-  public static void gatherSubReaders(List<IndexReader> allSubReaders, IndexReader
reader) {
-    IndexReader[] subReaders = reader.getSequentialSubReaders();
-    if (subReaders == null) {
-      // Add the reader itself, and do not recurse
-      allSubReaders.add(reader);
-    } else {
-      for (int i = 0; i < subReaders.length; i++) {
-        gatherSubReaders(allSubReaders, subReaders[i]);
-      }
+
+  public static void gatherSubReaders(final List<IndexReader> allSubReaders, IndexReader
reader) {
+    try {
+      new Gather(reader) {
+        @Override
+          protected void add(int base, IndexReader r) {
+          allSubReaders.add(r);
+        }
+      }.run();
+    } catch (IOException ioe) {
+      // won't happen
+      throw new RuntimeException(ioe);
     }
   }
 
-  public static int gatherSubFields(List<IndexReader> readers, List<Fields> fields,
List<Slice> slices, IndexReader reader, int base) throws IOException {
-    IndexReader[] subReaders = reader.getSequentialSubReaders();
-    if (subReaders == null) {
-      // Add the reader's fields
-      if (readers != null) {
-        readers.add(reader);
-      }
-      fields.add(reader.fields());
-      slices.add(new Slice(base, reader.maxDoc(), fields.size()-1));
-      base += reader.maxDoc();
-    } else {
-      for (int i = 0; i < subReaders.length; i++) {
-        base = gatherSubFields(readers, fields, slices, subReaders[i], base);
+  public static abstract class Gather {
+    private final IndexReader topReader;
+
+    public Gather(IndexReader r) {
+      topReader = r;
+    }
+
+    public int run() throws IOException {
+      return run(0, topReader);
+    }
+
+    public int run(int docBase) throws IOException {
+      return run(docBase, topReader);
+    }
+
+    private int run(int base, IndexReader reader) throws IOException {
+      IndexReader[] subReaders = reader.getSequentialSubReaders();
+      if (subReaders == null) {
+        // atomic reader
+        add(base, reader);
+        base += reader.maxDoc();
+      } else {
+        // composite reader
+        for (int i = 0; i < subReaders.length; i++) {
+          base = run(base, subReaders[i]);
+        }
       }
+
+      return base;
     }
 
-    return base;
+    protected abstract void add(int base, IndexReader r) throws IOException;
   }
 
   /**

Modified: lucene/java/branches/flex_1458/src/test/org/apache/lucene/TestExternalCodecs.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/test/org/apache/lucene/TestExternalCodecs.java?rev=920378&r1=920377&r2=920378&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/test/org/apache/lucene/TestExternalCodecs.java (original)
+++ lucene/java/branches/flex_1458/src/test/org/apache/lucene/TestExternalCodecs.java Mon
Mar  8 16:10:31 2010
@@ -37,8 +37,7 @@
   // For fun, test that we can override how terms are
   // sorted, and basic things still work -- this comparator
   // sorts in reversed unicode code point order:
-  private static final BytesRef.Comparator reverseUnicodeComparator = new BytesRef.Comparator()
{
-      @Override
+  private static final Comparator<BytesRef> reverseUnicodeComparator = new Comparator<BytesRef>()
{
       public int compare(BytesRef t1, BytesRef t2) {
         byte[] b1 = t1.bytes;
         byte[] b2 = t2.bytes;
@@ -62,6 +61,10 @@
         // One is prefix of another, or they are equal
         return t2.length-t1.length;
       }
+
+      public boolean equals(Object other) {
+        return this == other;
+      }
     };
 
   // TODO
@@ -110,7 +113,7 @@
       }
 
       @Override
-      public BytesRef.Comparator getComparator() {
+      public Comparator<BytesRef> getComparator() {
         return reverseUnicodeComparator;
       }
     }
@@ -175,17 +178,15 @@
 
       
       @Override
-      public BytesRef.Comparator getComparator() {
+      public Comparator<BytesRef> getComparator() {
         return BytesRef.getUTF8SortedAsUTF16Comparator();
       }
 
       @Override
       public void finishTerm(BytesRef text, int numDocs) {
-        // nocommit -- are we even called when numDocs == 0?
-        if (numDocs > 0) {
-          assert numDocs == current.docs.size();
-          field.termToDocs.put(current.term, current);
-        }
+        assert numDocs > 0;
+        assert numDocs == current.docs.size();
+        field.termToDocs.put(current.term, current);
       }
 
       @Override
@@ -203,7 +204,7 @@
       }
 
       @Override
-      public void addDoc(int docID, int freq) {
+      public void startDoc(int docID, int freq) {
         current = new RAMDoc(docID, freq);
         term.docs.add(current);
         posUpto = 0;
@@ -261,7 +262,7 @@
       }
       
       @Override
-      public BytesRef.Comparator getComparator() {
+      public Comparator<BytesRef> getComparator() {
         return BytesRef.getUTF8SortedAsUTF16Comparator();
       }
 
@@ -285,10 +286,10 @@
       @Override
       public SeekStatus seek(BytesRef term) {
         current = term.utf8ToString();
+        it = null;
         if (ramField.termToDocs.containsKey(current)) {
           return SeekStatus.FOUND;
         } else {
-          // nocommit -- right?
           if (current.compareTo(ramField.termToDocs.lastKey()) > 0) {
             return SeekStatus.END;
           } else {
@@ -465,11 +466,11 @@
     }
 
     @Override
-    public void getExtensions(Collection extensions) {
+    public void getExtensions(Set<String> extensions) {
     }
 
     @Override
-    public void files(Directory dir, SegmentInfo segmentInfo, Collection files) {
+    public void files(Directory dir, SegmentInfo segmentInfo, Set<String> files) {
     }
   }
 
@@ -477,6 +478,7 @@
    *  You must ensure every field you index has a Codec, or
    *  the defaultCodec is non null.  Also, the separate
    *  codecs cannot conflict on file names.*/
+  // nocommit -- promote to core
   public static class PerFieldCodecWrapper extends Codec {
     private final Map<String,Codec> fields = new HashMap<String,Codec>();
     private final Codec defaultCodec;
@@ -523,16 +525,26 @@
           fields = codec.fieldsConsumer(state);
           codecs.put(codec, fields);
         }
-        //System.out.println("field " + field.name + " -> codec " + codec);
         return fields.addField(field);
       }
 
       @Override
       public void close() throws IOException {
         Iterator<FieldsConsumer> it = codecs.values().iterator();
+        IOException err = null;
         while(it.hasNext()) {
-          // nocommit -- catch exc and keep closing the rest?
-          it.next().close();
+          try {
+            it.next().close();
+          } catch (IOException ioe) {
+            // keep first IOException we hit but keep
+            // closing the rest
+            if (err == null) {
+              err = ioe;
+            }
+          }
+        }
+        if (err != null) {
+          throw err;
         }
       }
     }
@@ -606,9 +618,20 @@
       @Override
       public void close() throws IOException {
         Iterator<FieldsProducer> it = codecs.values().iterator();
+        IOException err = null;
         while(it.hasNext()) {
-          // nocommit -- catch exc and keep closing the rest?
-          it.next().close();
+          try {
+            it.next().close();
+          } catch (IOException ioe) {
+            // keep first IOException we hit but keep
+            // closing the rest
+            if (err == null) {
+              err = ioe;
+            }
+          }
+        }
+        if (err != null) {
+          throw err;
         }
       }
 
@@ -616,7 +639,6 @@
       public void loadTermsIndex(int indexDivisor) throws IOException {
         Iterator<FieldsProducer> it = codecs.values().iterator();
         while(it.hasNext()) {
-          // nocommit -- catch exc and keep closing the rest?
           it.next().loadTermsIndex(indexDivisor);
         }
       }
@@ -630,7 +652,7 @@
     }
 
     @Override
-    public void files(Directory dir, SegmentInfo info, Collection<String> files) throws
IOException {
+    public void files(Directory dir, SegmentInfo info, Set<String> files) throws IOException
{
       Iterator<Codec> it = fields.values().iterator();
       Set<Codec> seen = new HashSet<Codec>();
       while(it.hasNext()) {
@@ -643,7 +665,7 @@
     }
 
     @Override
-    public void getExtensions(Collection<String> extensions) {
+    public void getExtensions(Set<String> extensions) {
       Iterator<Codec> it = fields.values().iterator();
       while(it.hasNext()) {
         final Codec codec = it.next();
@@ -762,14 +784,14 @@
     }
 
     @Override
-    public void files(Directory dir, SegmentInfo segmentInfo, Collection<String> files)
throws IOException {
+    public void files(Directory dir, SegmentInfo segmentInfo, Set<String> files) throws
IOException {
       StandardPostingsReaderImpl.files(dir, segmentInfo, files);
       StandardTermsDictReader.files(dir, segmentInfo, files);
       SimpleStandardTermsIndexReader.files(dir, segmentInfo, files);
     }
 
     @Override
-    public void getExtensions(Collection<String> extensions) {
+    public void getExtensions(Set<String> extensions) {
       StandardCodec.getStandardExtensions(extensions);
     }
   }

Modified: lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/FlexTestUtil.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/FlexTestUtil.java?rev=920378&r1=920377&r2=920378&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/FlexTestUtil.java (original)
+++ lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/FlexTestUtil.java Mon
Mar  8 16:10:31 2010
@@ -25,7 +25,6 @@
 
 public class FlexTestUtil {
 
-  // nocommit:
   // index variations
   //   need del docs
   //   need payloads
@@ -73,8 +72,7 @@
   public static void verifyFlexVsPreFlex(Random rand, IndexReader r) throws Exception {
     // First test on DirReader
 
-    // nocommit turn back on
-    // verifyFlexVsPreFlexSingle(rand, r);
+    verifyFlexVsPreFlexSingle(rand, r);
 
     // Then on each individual sub reader
     IndexReader[] subReaders = r.getSequentialSubReaders();
@@ -86,24 +84,18 @@
     }
 
     // Then on a new MultiReader
-    // nocommit -- back on:
-    if (false) {
-      IndexReader m = new MultiReader(subReaders, false);
-      verifyFlexVsPreFlexSingle(rand, m);
-      m.close();
-    }
+    IndexReader m = new MultiReader(subReaders, false);
+    verifyFlexVsPreFlexSingle(rand, m);
+    m.close();
 
     // Then on a forced-external reader (forced flex to
     // emulate API on pre-flex API, which in turn is
     // emulating pre-flex on flex -- twisted, but, better
     // work):
-    // nocommit back on
-    if (false) {
-      verifyFlexVsPreFlexSingle(rand, new ForcedExternalReader(r));
-      IndexReader m = new MultiReader(forcedSubReaders, false);
-      verifyFlexVsPreFlexSingle(rand, m);
-      m.close();
-    }
+    verifyFlexVsPreFlexSingle(rand, new ForcedExternalReader(r));
+    m = new MultiReader(forcedSubReaders, false);
+    verifyFlexVsPreFlexSingle(rand, m);
+    m.close();
   }
 
   private static void verifyFlexVsPreFlexSingle(Random rand, IndexReader r) throws Exception
{
@@ -119,9 +111,13 @@
 
     // straight enum of fields/terms/docs/positions
     TermEnum termEnum = r.terms();
-    FieldsEnum fields = r.fields().iterator();
+    final Fields fields = MultiFields.getFields(r);
+    if (fields == null) {
+      return;
+    }
+    FieldsEnum fieldsEnum = fields.iterator();
     while(true) {
-      final String field = fields.next();
+      final String field = fieldsEnum.next();
       if (field == null) {
         boolean result = termEnum.next();
         if (result) {
@@ -130,7 +126,7 @@
         assertFalse(result);
         break;
       }
-      TermsEnum terms = fields.terms();
+      TermsEnum terms = fieldsEnum.terms();
       DocsAndPositionsEnum postings = null;
       DocsEnum docsEnum = null;
       final TermPositions termPos = r.termPositions();
@@ -146,8 +142,8 @@
           assertEquals(termEnum.docFreq(), terms.docFreq());
           //allTerms.add(t);
 
-          postings = terms.docsAndPositions(r.getDeletedDocs(), postings);
-          docsEnum = terms.docs(r.getDeletedDocs(), docsEnum);
+          postings = terms.docsAndPositions(MultiFields.getDeletedDocs(r), postings);
+          docsEnum = terms.docs(MultiFields.getDeletedDocs(r), docsEnum);
 
           final DocsEnum docs;
           if (postings != null) {
@@ -166,16 +162,18 @@
               assertTrue(termPos.next());
               assertEquals(termPos.doc(), doc);
               assertEquals(termPos.freq(), docs.freq());
-              //System.out.println("TEST:     doc=" + doc + " freq=" + docs.freq());
               final int freq = docs.freq();
               if (postings == null) {
                 assertEquals(1, freq);
-                assertEquals(0, termPos.nextPosition());
+                // Old API did not always do this,
+                // specifically in the MultiTermPositions
+                // case when some segs omit positions and
+                // some don't
+                //assertEquals(0, termPos.nextPosition());
                 assertEquals(false, termPos.isPayloadAvailable());
               } else {
                 for(int i=0;i<freq;i++) {
                   final int position = postings.nextPosition();
-                  //System.out.println("TEST:       pos=" + position);
                   assertEquals(position, termPos.nextPosition());
                   assertEquals(postings.hasPayload(), termPos.isPayloadAvailable());
                   if (postings.hasPayload()) {
@@ -198,15 +196,18 @@
   private static void testRandomSkips(Random rand, IndexReader r) throws Exception {
 
     TermEnum termEnum = r.terms();
-    FieldsEnum fields = r.fields().iterator();
+    Fields fields = MultiFields.getFields(r);
+    if (fields == null) {
+      return;
+    }
+    FieldsEnum fieldsEnum = fields.iterator();
     boolean skipNext = false;
     int[] docs1 = new int[16];
     int[] freqs1 = new int[16];
     int[] docs2 = new int[16];
     int[] freqs2 = new int[16];
     while(true) {
-      final String field = fields.next();
-      //System.out.println("TEST: enum field=" + field);
+      final String field = fieldsEnum.next();
       if (field == null) {
         boolean result = termEnum.next();
         if (result) {
@@ -218,14 +219,13 @@
       if (rand.nextInt(3) <= 1) {
         // Enum the terms
         //System.out.println("TEST:   get terms");
-        TermsEnum terms = fields.terms();
+        TermsEnum terms = fieldsEnum.terms();
         final TermPositions termPos = r.termPositions();
         final TermDocs termDocs = r.termDocs();
         DocsEnum docs = null;
         DocsAndPositionsEnum postings = null;
         while(true) {
           final BytesRef termRef = terms.next();
-          //System.out.println("TEST:   enum term=" + termRef);
           if (termRef == null) {
             break;
           } else {
@@ -241,9 +241,9 @@
             //allTerms.add(t);
 
             if (rand.nextInt(3) <= 1) {
-              docs = terms.docs(r.getDeletedDocs(), docs);
+              docs = terms.docs(MultiFields.getDeletedDocs(r), docs);
               assert !(docs instanceof DocsAndPositionsEnum): "docs=" + docs;
-              postings = terms.docsAndPositions(r.getDeletedDocs(), postings);
+              postings = terms.docsAndPositions(MultiFields.getDeletedDocs(r), postings);
               final DocsEnum docsEnum;
               if (postings == null) {
                 docsEnum = docs;
@@ -253,17 +253,39 @@
               if (rand.nextBoolean()) {
                 // use bulk read API
                 termDocs.seek(t);
+                int count1 = 0;
+                int count2 = 0;
                 while(true) {
-                  final int count1 = docs.read(docs1, freqs1);
-                  final int count2 = termDocs.read(docs2, freqs2);
-                  assertEquals(count1, count2);
                   if (count1 == 0) {
+                    count1 = docs.read(docs1, freqs1);
+                  }
+                  if (count2 == 0) {
+                    count2 = termDocs.read(docs2, freqs2);
+                  }
+
+                  if (count1 == 0 || count2 == 0) {
+                    assertEquals(0, count2);
+                    assertEquals(0, count1);
                     break;
                   }
-                  for(int i=0;i<count1;i++) {
+                  final int limit = Math.min(count1, count2);
+                  for(int i=0;i<limit;i++) {
                     assertEquals(docs1[i], docs2[i]);
                     assertEquals(freqs1[i], freqs2[i]);
                   }
+                  if (count1 > limit) {
+                    // copy down
+                    System.arraycopy(docs1, limit, docs1, 0, count1-limit);
+                    System.arraycopy(freqs1, limit, freqs1, 0, count1-limit);
+                  }
+                  count1 -= limit;
+
+                  if (count2 > limit) {
+                    // copy down
+                    System.arraycopy(docs2, limit, docs2, 0, count2-limit);
+                    System.arraycopy(freqs2, limit, freqs2, 0, count2-limit);
+                  }
+                  count2 -= limit;
                 }
               } else {
                 // Enum the docs one by one
@@ -284,7 +306,11 @@
                       final int freq = docsEnum.freq();
                       if (postings == null) {
                         assertEquals(1, termPos.freq());
-                        assertEquals(0, termPos.nextPosition());
+                        // Old API did not always do this,
+                        // specifically in the MultiTermPositions
+                        // case when some segs omit positions and
+                        // some don't
+                        //assertEquals(0, termPos.nextPosition());
                         assertFalse(termPos.isPayloadAvailable());
                       } else {
                         // we have positions
@@ -387,9 +413,13 @@
   private static void testRandomSeeks(Random rand, IndexReader r) throws Exception {
     final int ITER = 100;
     List<String> allFields = new ArrayList<String>();
-    FieldsEnum fields = r.fields().iterator();
+    Fields fields = MultiFields.getFields(r);
+    if (fields == null) {
+      return;
+    }
+    FieldsEnum fieldsEnum = fields.iterator();
     while(true) {
-      String f = fields.next();
+      String f = fieldsEnum.next();
       if (f == null) {
         break;
       }
@@ -409,7 +439,7 @@
       String f = allFields.get(rand.nextInt(fieldCount));
 
       String text = getRandomText(rand, 1, 3, false);
-      final TermsEnum termsEnum = r.fields().terms(f).iterator();
+      final TermsEnum termsEnum = MultiFields.getFields(r).terms(f).iterator();
 
       final TermsEnum.SeekStatus seekStatus = termsEnum.seek(new BytesRef(text));
       Term t = new Term(f, text);
@@ -431,8 +461,8 @@
 
       assertEquals(termsEnum.docFreq(), termEnum.docFreq());
 
-      docs = termsEnum.docs(r.getDeletedDocs(), docs);
-      postings = termsEnum.docsAndPositions(r.getDeletedDocs(), postings);
+      docs = termsEnum.docs(MultiFields.getDeletedDocs(r), docs);
+      postings = termsEnum.docsAndPositions(MultiFields.getDeletedDocs(r), postings);
 
       termPositions.seek(termEnum.term());
 
@@ -465,7 +495,11 @@
 
         if (postings == null) {
           assertEquals(1, termPositions.freq());
-          assertEquals(0, termPositions.nextPosition());
+          // Old API did not always do this,
+          // specifically in the MultiTermPositions
+          // case when some segs omit positions and
+          // some don't
+          //assertEquals(0, termPositions.nextPosition());
           assertFalse(termPositions.isPayloadAvailable());
         } else {
           for(int k=0;k<docsEnum.freq();k++) {
@@ -489,7 +523,8 @@
 
   // Delegates to a "normal" IndexReader, making it look
   // "external", to force testing of the "flex API on
-  // external reader" layer
+  // external reader" layer.  DO NOT OVERRIDE
+  // getSequentialSubReaders!!
   public final static class ForcedExternalReader extends IndexReader {
     private final IndexReader r;
     public ForcedExternalReader(IndexReader r) {
@@ -513,7 +548,7 @@
     }
 
     public Bits getDeletedDocs() throws IOException {
-      return r.getDeletedDocs();
+      return MultiFields.getDeletedDocs(r);
     }
 
     public int numDocs() {
@@ -540,6 +575,10 @@
       return r.norms(field);
     }
 
+    public String toString() {
+      return "ForcedExternalReader(" + r + ")";
+    }
+
     public void norms(String field, byte[] bytes, int offset) 
       throws IOException {
       r.norms(field, bytes, offset);

Modified: lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestCodecs.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestCodecs.java?rev=920378&r1=920377&r2=920378&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestCodecs.java (original)
+++ lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestCodecs.java Mon Mar
 8 16:10:31 2010
@@ -23,7 +23,7 @@
 import org.apache.lucene.store.*;
 import java.util.*;
 
-// nocommit -- test multiple codecs here?
+// TODO: test multiple codecs here?
 
 // TODO
 //   - test across fields
@@ -43,14 +43,11 @@
 
 public class TestCodecs extends LuceneTestCase {
 
-  // nocommit -- switch to newRandom():
-  private static final Random RANDOM = new Random(42);
+  private Random RANDOM;
   private static String[] fieldNames = new String[] {"one", "two", "three", "four"};
 
   private final static int NUM_TEST_ITER = 4000;
-  // nocommit
-  //private final static int NUM_TEST_THREADS = 3;
-  private final static int NUM_TEST_THREADS = 1;
+  private final static int NUM_TEST_THREADS = 3;
   private final static int NUM_FIELDS = 4;
   private final static int NUM_TERMS_RAND = 50; // must be > 16 to test skipping
   private final static int DOC_FREQ_RAND = 500; // must be > 16 to test skipping
@@ -167,7 +164,7 @@
         } else {
           termDocFreq = positions[i].length;
         }
-        postingsConsumer.addDoc(docs[i], termDocFreq);
+        postingsConsumer.startDoc(docs[i], termDocFreq);
         if (!field.omitTF) {
           for(int j=0;j<positions[i].length;j++) {
             PositionData pos = positions[i][j];
@@ -248,6 +245,8 @@
 
   public void testFixedPostings() throws Throwable {
 
+    RANDOM = newRandom();
+
     final int NUM_TERMS = 100;
     TermData[] terms = new TermData[NUM_TERMS];
     for(int i=0;i<NUM_TERMS;i++) {
@@ -287,7 +286,7 @@
 
   public void testRandomPostings() throws Throwable {
 
-    // Codec.DEBUG = true;
+    RANDOM = newRandom();
 
     final FieldInfos fieldInfos = new FieldInfos();
     
@@ -384,7 +383,7 @@
               System.out.println("TEST do check payload len=" + posEnum.getPayloadLength()
+ " vs " + (otherPayload == null ? "null" : otherPayload.length));
             }
 
-            assertTrue("expected=" + positions[i].payload.toBytesString() + " got=" + otherPayload.toBytesString(),
positions[i].payload.equals(otherPayload));
+            assertTrue("expected=" + positions[i].payload.toString() + " got=" + otherPayload.toString(),
positions[i].payload.equals(otherPayload));
           } else {
             if (Codec.DEBUG) {
               System.out.println("TEST skip check payload len=" + posEnum.getPayloadLength());
@@ -528,9 +527,23 @@
                   System.out.println("TEST [" + getDesc(field, term) + "]: skip: " + left
+ " docs left; skip to doc=" + term.docs[upto2] + " [" + upto2 + " of " + term.docs.length
+ "]");
                 }
 
-                doc = docsEnum.advance(term.docs[upto2]);
-                // nocommit -- test skipping to non-existent doc
-                assertEquals(term.docs[upto2], doc);
+                if (nextInt(2) == 1) {
+                  doc = docsEnum.advance(term.docs[upto2]);
+                  assertEquals(term.docs[upto2], doc);
+                } else {
+                  doc = docsEnum.advance(1+term.docs[upto2]);
+                  if (doc == DocsEnum.NO_MORE_DOCS) {
+                    // skipped past last doc
+                    assert upto2 == term.docs.length-1;
+                    break;
+                  } else {
+                    // skipped to next doc
+                    assert upto2 < term.docs.length-1;
+                    if (doc >= term.docs[1+upto2]) {
+                      upto2++;
+                    }
+                  }
+                }
               } else {
                 doc = docsEnum.nextDoc();
                 assertTrue(doc != -1);
@@ -565,16 +578,13 @@
         } while (termsEnum.next() != null);
 
         assertEquals(upto, field.terms.length);
-        
-        //termsEnum.close();
       }
     }
   }
 
   private void write(FieldInfos fieldInfos, Directory dir, FieldData[] fields) throws Throwable
{
 
-    // nocommit -- randomize this:
-    final int termIndexInterval = 16;
+    final int termIndexInterval = nextInt(13, 27);
 
     SegmentWriteState state = new SegmentWriteState(null, dir, SEGMENT, fieldInfos, null,
10000, 10000, termIndexInterval,
                                                     Codecs.getDefault());

Modified: lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestIndexWriter.java?rev=920378&r1=920377&r2=920378&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestIndexWriter.java (original)
+++ lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestIndexWriter.java Mon
Mar  8 16:10:31 2010
@@ -4341,13 +4341,6 @@
       new IndexWriter(dir, new WhitespaceAnalyzer(TEST_VERSION_CURRENT), true, IndexWriter.MaxFieldLength.LIMITED).close();
 
       assertTrue(dir.fileExists("myrandomfile"));
-
-      // Make sure this does not copy myrandomfile:
-      // nocommit -- Directory.copy now copies all files --
-      // how to fix?
-      //Directory dir2 = new RAMDirectory(dir);
-      //assertTrue(!dir2.fileExists("myrandomfile"));
-
     } finally {
       dir.close();
       _TestUtil.rmDir(indexDir);

Modified: lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestLazyProxSkipping.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestLazyProxSkipping.java?rev=920378&r1=920377&r2=920378&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestLazyProxSkipping.java
(original)
+++ lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestLazyProxSkipping.java
Mon Mar  8 16:10:31 2010
@@ -50,7 +50,6 @@
         IndexInput ii = super.openInput(name);
         if (name.endsWith(".prx") || name.endsWith(".pos") ) {
           // we decorate the proxStream with a wrapper class that allows to count the number
of calls of seek()
-          // nocommit -- fix this:
           ii = new SeeksCountingStream(ii);
         }
         return ii;

Modified: lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestOmitTf.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestOmitTf.java?rev=920378&r1=920377&r2=920378&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestOmitTf.java (original)
+++ lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestOmitTf.java Mon Mar
 8 16:10:31 2010
@@ -80,7 +80,7 @@
     // keep things constant
     d = new Document();
         
-    // Reverese
+    // Reverse
     f1.setOmitTermFreqAndPositions(true);
     d.add(f1);
         
@@ -90,8 +90,9 @@
     Random rnd = newRandom();
 
     writer.addDocument(d);
-    // force merge
     FlexTestUtil.verifyFlexVsPreFlex(rnd, writer);
+
+    // force merge
     writer.optimize();
     // flush
     writer.close();

Modified: lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestSegmentTermEnum.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestSegmentTermEnum.java?rev=920378&r1=920377&r2=920378&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestSegmentTermEnum.java
(original)
+++ lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestSegmentTermEnum.java
Mon Mar  8 16:10:31 2010
@@ -23,7 +23,6 @@
 import org.apache.lucene.analysis.WhitespaceAnalyzer;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
-import org.apache.lucene.index.codecs.preflex.SegmentTermEnum;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.RAMDirectory;
 import org.apache.lucene.store.MockRAMDirectory;
@@ -61,8 +60,6 @@
     verifyDocFreq();
   }
 
-  // nocommit
-  /*
   public void testPrevTermAtEnd() throws IOException
   {
     Directory dir = new MockRAMDirectory();
@@ -70,16 +67,18 @@
     addDoc(writer, "aaa bbb");
     writer.close();
     SegmentReader reader = SegmentReader.getOnlySegmentReader(dir);
-    SegmentTermEnum termEnum = (SegmentTermEnum) reader.terms();
-    assertTrue(termEnum.next());
-    assertEquals("aaa", termEnum.term().text());
-    assertTrue(termEnum.next());
-    assertEquals("aaa", termEnum.prev().text());
-    assertEquals("bbb", termEnum.term().text());
-    assertFalse(termEnum.next());
-    assertEquals("bbb", termEnum.prev().text());
+    TermsEnum terms = reader.fields().terms("content").iterator();
+    assertNotNull(terms.next());
+    assertEquals("aaa", terms.term().utf8ToString());
+    assertNotNull(terms.next());
+    long ordB = terms.ord();
+    assertEquals("bbb", terms.term().utf8ToString());
+    assertNull(terms.next());
+
+    assertEquals(TermsEnum.SeekStatus.FOUND, terms.seek(ordB));
+    assertEquals("bbb", terms.term().utf8ToString());
   }
-  */
+
   private void verifyDocFreq()
       throws IOException
   {

Modified: lucene/java/branches/flex_1458/src/test/org/apache/lucene/search/TestBoolean2.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/test/org/apache/lucene/search/TestBoolean2.java?rev=920378&r1=920377&r2=920378&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/test/org/apache/lucene/search/TestBoolean2.java (original)
+++ lucene/java/branches/flex_1458/src/test/org/apache/lucene/search/TestBoolean2.java Mon
Mar  8 16:10:31 2010
@@ -198,7 +198,6 @@
   }
 
   public void testRandomQueries() throws Exception {
-    // nocommit -- remove 17 seed
     Random rnd = newRandom();
 
     String[] vals = {"w1","w2","w3","w4","w5","xx","yy","zzz"};



Mime
View raw message