lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From uschind...@apache.org
Subject svn commit: r1293965 - in /lucene/dev/trunk/lucene: core/src/test/org/apache/lucene/index/ test-framework/src/java/org/apache/lucene/index/ test-framework/src/java/org/apache/lucene/search/ test-framework/src/java/org/apache/lucene/util/
Date Sun, 26 Feb 2012 23:38:01 GMT
Author: uschindler
Date: Sun Feb 26 23:38:00 2012
New Revision: 1293965

URL: http://svn.apache.org/viewvc?rev=1293965&view=rev
Log:
LUCENE-3823: Add a field-filtering FilterAtomicReader to 4.0 so ParallelReaders can be better
tested (in LTC.maybeWrapReader)

Added:
    lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/FieldFilterAtomicReader.java
  (with props)
Modified:
    lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestDuelingCodecs.java
    lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestTypePromotion.java
    lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/search/QueryUtils.java
    lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java

Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestDuelingCodecs.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestDuelingCodecs.java?rev=1293965&r1=1293964&r2=1293965&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestDuelingCodecs.java (original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestDuelingCodecs.java Sun
Feb 26 23:38:00 2012
@@ -18,6 +18,8 @@ package org.apache.lucene.index;
  */
 
 import java.io.IOException;
+import java.util.Collections;
+import java.util.Comparator;
 import java.util.HashSet;
 import java.util.Iterator;
 import java.util.Random;
@@ -29,7 +31,6 @@ import org.apache.lucene.analysis.MockAn
 import org.apache.lucene.codecs.Codec;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.search.DocIdSetIterator;
-import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.BytesRef;
@@ -97,10 +98,9 @@ public class TestDuelingCodecs extends L
     createRandomIndex(numdocs, leftWriter, seed);
     createRandomIndex(numdocs, rightWriter, seed);
 
-    // TODO: maybe we should do this wrapping in another test?
-    leftReader = maybeWrap(leftWriter.getReader());
+    leftReader = maybeWrapReader(leftWriter.getReader());
     leftWriter.close();
-    rightReader = maybeWrap(rightWriter.getReader());
+    rightReader = maybeWrapReader(rightWriter.getReader());
     rightWriter.close();
     
     info = "left: " + leftCodec.toString() + " / right: " + rightCodec.toString();
@@ -116,12 +116,6 @@ public class TestDuelingCodecs extends L
     super.tearDown();
   }
   
-  static IndexReader maybeWrap(IndexReader other) throws IOException {
-    // TODO: bogus how we do this
-    IndexSearcher is = newSearcher(other);
-    return is.getIndexReader();
-  }
-  
   /**
    * populates a writer with random stuff. this must be fully reproducable with the seed!
    */
@@ -488,8 +482,18 @@ public class TestDuelingCodecs extends L
       Document rightDoc = rightReader.document(i);
       
       // TODO: I think this is bogus because we don't document what the order should be
-      // from these iterators, etc. I think the codec should be free to order this stuff
+      // from these iterators, etc. I think the codec/IndexReader should be free to order
this stuff
       // in whatever way it wants (e.g. maybe it packs related fields together or something)
+      // To fix this, we sort the fields in both documents by name, but
+      // we still assume that all instances with same name are in order:
+      Comparator<IndexableField> comp = new Comparator<IndexableField>() {
+        @Override
+        public int compare(IndexableField arg0, IndexableField arg1) {
+          return arg0.name().compareTo(arg1.name());
+        }        
+      };
+      Collections.sort(leftDoc.getFields(), comp);
+      Collections.sort(rightDoc.getFields(), comp);
 
       Iterator<IndexableField> leftIterator = leftDoc.iterator();
       Iterator<IndexableField> rightIterator = rightDoc.iterator();

Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestTypePromotion.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestTypePromotion.java?rev=1293965&r1=1293964&r2=1293965&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestTypePromotion.java (original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestTypePromotion.java Sun
Feb 26 23:38:00 2012
@@ -97,9 +97,8 @@ public class TestTypePromotion extends L
         writer.addIndexes(dir_2);
       } else {
         // do a real merge here
-        IndexReader open = IndexReader.open(dir_2);
-        // TODO: wrap in a better way
-        writer.addIndexes(newSearcher(open).getIndexReader());
+        IndexReader open = maybeWrapReader(IndexReader.open(dir_2));
+        writer.addIndexes(open);
         open.close();
       }
       dir_2.close();

Added: lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/FieldFilterAtomicReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/FieldFilterAtomicReader.java?rev=1293965&view=auto
==============================================================================
--- lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/FieldFilterAtomicReader.java
(added)
+++ lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/FieldFilterAtomicReader.java
Sun Feb 26 23:38:00 2012
@@ -0,0 +1,169 @@
+package org.apache.lucene.index;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.util.Set;
+
+import org.apache.lucene.index.FilterAtomicReader;
+
+public final class FieldFilterAtomicReader extends FilterAtomicReader {
+  
+  private final Set<String> fields;
+  private final boolean negate;
+  private final FieldInfos fieldInfos;
+
+  public FieldFilterAtomicReader(AtomicReader in, Set<String> fields, boolean negate)
{
+    super(in);
+    this.fields = fields;
+    this.negate = negate;
+    this.fieldInfos = new FieldInfos();
+    for (FieldInfo fi : in.getFieldInfos()) {
+      if (hasField(fi.name)) {
+        fieldInfos.add(fi);
+      }
+    }
+  }
+  
+  boolean hasField(String field) {
+    return negate ^ fields.contains(field);
+  }
+
+  @Override
+  public FieldInfos getFieldInfos() {
+    return fieldInfos;
+  }
+
+  @Override
+  public Fields getTermVectors(int docID) throws IOException {
+    Fields f = super.getTermVectors(docID);
+    if (f == null) {
+      return null;
+    }
+    f = new FieldFilterFields(f);
+    // we need to check for emptyness, so we can return null:
+    return (f.iterator().next() == null) ? null : f;
+  }
+
+  @Override
+  public void document(final int docID, final StoredFieldVisitor visitor) throws CorruptIndexException,
IOException {
+    super.document(docID, new StoredFieldVisitor() {
+      @Override
+      public void binaryField(FieldInfo fieldInfo, byte[] value, int offset, int length)
throws IOException {
+        visitor.binaryField(fieldInfo, value, offset, length);
+      }
+
+      @Override
+      public void stringField(FieldInfo fieldInfo, String value) throws IOException {
+        visitor.stringField(fieldInfo, value);
+      }
+
+      @Override
+      public void intField(FieldInfo fieldInfo, int value) throws IOException {
+        visitor.intField(fieldInfo, value);
+      }
+
+      @Override
+      public void longField(FieldInfo fieldInfo, long value) throws IOException {
+        visitor.longField(fieldInfo, value);
+      }
+
+      @Override
+      public void floatField(FieldInfo fieldInfo, float value) throws IOException {
+        visitor.floatField(fieldInfo, value);
+      }
+
+      @Override
+      public void doubleField(FieldInfo fieldInfo, double value) throws IOException {
+        visitor.doubleField(fieldInfo, value);
+      }
+
+      @Override
+      public Status needsField(FieldInfo fieldInfo) throws IOException {
+        return hasField(fieldInfo.name) ? visitor.needsField(fieldInfo) : Status.NO;
+      }
+    });
+  }
+
+  @Override
+  public boolean hasNorms(String field) throws IOException {
+    return hasField(field) ? super.hasNorms(field) : false;
+  }
+
+  @Override
+  public Fields fields() throws IOException {
+    final Fields f = super.fields();
+    return (f == null) ? null : new FieldFilterFields(f);
+  }
+
+  @Override
+  public DocValues docValues(String field) throws IOException {
+    return hasField(field) ? super.docValues(field) : null;
+  }
+
+  @Override
+  public DocValues normValues(String field) throws IOException {
+    return hasField(field) ? super.normValues(field) : null;
+  }
+
+  @Override
+  public String toString() {
+    final StringBuilder sb = new StringBuilder("FieldFilterAtomicReader(reader=");
+    sb.append(in).append(", fields=");
+    if (negate) sb.append('!');
+    return sb.append(fields).append(')').toString();
+  }
+  
+  private class FieldFilterFields extends FilterFields {
+    public FieldFilterFields(Fields in) {
+      super(in);
+    }
+
+    @Override
+    public int getUniqueFieldCount() throws IOException {
+      // TODO: add faster implementation!
+      int c = 0;
+      final FieldsEnum it = iterator();
+      while (it.next() != null) {
+        c++;
+      }
+      return c;
+    }
+
+    @Override
+    public FieldsEnum iterator() throws IOException {
+      return new FilterFieldsEnum(super.iterator()) {
+        @Override
+        public String next() throws IOException {
+          String f;
+          while ((f = super.next()) != null) {
+            if (hasField(f)) return f;
+          }
+          return null;
+        } 
+      };
+    }
+
+    @Override
+    public Terms terms(String field) throws IOException {
+      return hasField(field) ? super.terms(field) : null;
+    }
+    
+  }
+  
+}

Modified: lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/search/QueryUtils.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/search/QueryUtils.java?rev=1293965&r1=1293964&r2=1293965&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/search/QueryUtils.java
(original)
+++ lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/search/QueryUtils.java
Sun Feb 26 23:38:00 2012
@@ -167,10 +167,7 @@ public class QueryUtils {
           0 < edge ? r : emptyReaders[0])
     };
 
-    // TODO: fix me,
-    //   wrapping causes insanity when we have an already-atomic reader?!
-    // IndexSearcher out = LuceneTestCase.newSearcher(new MultiReader(readers));
-    IndexSearcher out = LuceneTestCase.newSearcher(new MultiReader(readers), false);
+    IndexSearcher out = LuceneTestCase.newSearcher(new MultiReader(readers));
     out.setSimilarity(s.getSimilarity());
     return out;
   }

Modified: lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java?rev=1293965&r1=1293964&r2=1293965&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
(original)
+++ lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
Sun Feb 26 23:38:00 2012
@@ -55,8 +55,8 @@ import org.apache.lucene.document.Field;
 import org.apache.lucene.document.FieldType;
 import org.apache.lucene.index.AtomicReader;
 import org.apache.lucene.index.CompositeReader;
-import org.apache.lucene.index.FilterAtomicReader;
-import org.apache.lucene.index.Fields;
+import org.apache.lucene.index.FieldFilterAtomicReader;
+import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.MultiReader;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexReader;
@@ -1349,45 +1349,56 @@ public abstract class LuceneTestCase ext
   
   /** Sometimes wrap the IndexReader as slow, parallel or filter reader (or combinations
of that) */
   public static IndexReader maybeWrapReader(IndexReader r) throws IOException {
-    // TODO: remove this, and fix those tests to wrap before putting slow around:
-    final boolean wasOriginallyAtomic = r instanceof AtomicReader;
     if (rarely()) {
+      // TODO: remove this, and fix those tests to wrap before putting slow around:
+      final boolean wasOriginallyAtomic = r instanceof AtomicReader;
       for (int i = 0, c = random.nextInt(6)+1; i < c; i++) {
         switch(random.nextInt(4)) {
           case 0:
             r = SlowCompositeReaderWrapper.wrap(r);
             break;
           case 1:
+            // will create no FC insanity as Parallel*Reader has own cache key:
             r = (r instanceof AtomicReader) ?
               new ParallelAtomicReader((AtomicReader) r) :
               new ParallelCompositeReader((CompositeReader) r);
             break;
           case 2:
-            if (!wasOriginallyAtomic) { // dont wrap originally atomic readers to be composite
(some tests don't like)
-              r = new MultiReader(r);
-            }
+            // Häckidy-Hick-Hack: this will create FC insanity, so we patch MultiReader
to
+            // return a fake cache key, so insanity checker cannot walk along our reader:
+            r = new MultiReader(r) {
+              private final Object cacheKey = new Object();
+              @Override public Object getCoreCacheKey() { return cacheKey; }
+              @Override public Object getCombinedCoreAndDeletesKey() { return cacheKey; }
+              @Override public String toString() { return "MultiReader(" + subReaders[0]
+ ")"; }
+            };
             break;
           case 3:
-            if (r instanceof AtomicReader) {
-              r = new FilterAtomicReader((AtomicReader) r) {
-                @Override
-                public Fields fields() throws IOException {
-                  Fields f = super.fields();
-                  if (f == null) {
-                    return null;
-                  } else {
-                    return new FilterFields(f);
-                  }
-                }
-              };
+            final AtomicReader ar = SlowCompositeReaderWrapper.wrap(r);
+            final List<String> allFields = new ArrayList<String>();
+            for (FieldInfo fi : ar.getFieldInfos()) {
+              allFields.add(fi.name);
             }
+            Collections.shuffle(allFields, random);
+            final int end = allFields.isEmpty() ? 0 : random.nextInt(allFields.size());
+            final Set<String> fields = new HashSet<String>(allFields.subList(0,
end));
+            // will create no FC insanity as ParallelAtomicReader has own cache key:
+            r = new ParallelAtomicReader(
+              new FieldFilterAtomicReader(ar, fields, false),
+              new FieldFilterAtomicReader(ar, fields, true)
+            );
             break;
           default:
             fail("should not get here");
         }
       }
+      if (wasOriginallyAtomic) {
+        r = SlowCompositeReaderWrapper.wrap(r);
+      }
+      if (VERBOSE) {
+        System.out.println("maybeWrapReader wrapped: " +r);
+      }
     }
-    //System.out.println(r);
     return r;
   }
 



Mime
View raw message