lucene-java-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From yo...@apache.org
Subject svn commit: r329526 - in /lucene/java/trunk/src/test/org/apache/lucene/index: DocHelper.java TestDocumentWriter.java TestFieldInfos.java TestFieldsReader.java TestMultiReader.java TestSegmentMerger.java TestSegmentReader.java
Date Sun, 30 Oct 2005 05:42:14 GMT
Author: yonik
Date: Sat Oct 29 22:42:09 2005
New Revision: 329526

URL: http://svn.apache.org/viewcvs?rev=329526&view=rev
Log:
omitNorms tests, make some tests more generic

Modified:
    lucene/java/trunk/src/test/org/apache/lucene/index/DocHelper.java
    lucene/java/trunk/src/test/org/apache/lucene/index/TestDocumentWriter.java
    lucene/java/trunk/src/test/org/apache/lucene/index/TestFieldInfos.java
    lucene/java/trunk/src/test/org/apache/lucene/index/TestFieldsReader.java
    lucene/java/trunk/src/test/org/apache/lucene/index/TestMultiReader.java
    lucene/java/trunk/src/test/org/apache/lucene/index/TestSegmentMerger.java
    lucene/java/trunk/src/test/org/apache/lucene/index/TestSegmentReader.java

Modified: lucene/java/trunk/src/test/org/apache/lucene/index/DocHelper.java
URL: http://svn.apache.org/viewcvs/lucene/java/trunk/src/test/org/apache/lucene/index/DocHelper.java?rev=329526&r1=329525&r2=329526&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/index/DocHelper.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/index/DocHelper.java Sat Oct 29 22:42:09
2005
@@ -36,17 +36,28 @@
   public static final int [] FIELD_2_FREQS = {3, 1, 1}; 
   public static final String TEXT_FIELD_2_KEY = "textField2";
   public static Field textField2 = new Field(TEXT_FIELD_2_KEY, FIELD_2_TEXT, Field.Store.YES,
Field.Index.TOKENIZED, Field.TermVector.WITH_POSITIONS_OFFSETS);
-  
+
+  public static final String FIELD_3_TEXT = "aaaNoNorms aaaNoNorms bbbNoNorms";
+  public static final String TEXT_FIELD_3_KEY = "textField3";
+  public static Field textField3 = new Field(TEXT_FIELD_3_KEY, FIELD_3_TEXT, Field.Store.YES,
Field.Index.TOKENIZED);
+  static { textField3.setOmitNorms(true); }
+
   public static final String KEYWORD_TEXT = "Keyword";
   public static final String KEYWORD_FIELD_KEY = "keyField";
   public static Field keyField = new Field(KEYWORD_FIELD_KEY, KEYWORD_TEXT,
       Field.Store.YES, Field.Index.UN_TOKENIZED);
-  
+
+  public static final String NO_NORMS_TEXT = "omitNormsText";
+  public static final String NO_NORMS_KEY = "omitNorms";
+  public static Field noNormsField = new Field(NO_NORMS_KEY, NO_NORMS_TEXT,
+      Field.Store.YES, Field.Index.NO_NORMS);
+
   public static final String UNINDEXED_FIELD_TEXT = "unindexed field text";
   public static final String UNINDEXED_FIELD_KEY = "unIndField";
   public static Field unIndField = new Field(UNINDEXED_FIELD_KEY, UNINDEXED_FIELD_TEXT,
       Field.Store.YES, Field.Index.NO);
-  
+
+
   public static final String UNSTORED_1_FIELD_TEXT = "unstored field text";
   public static final String UNSTORED_FIELD_1_KEY = "unStoredField1";
   public static Field unStoredField1 = new Field(UNSTORED_FIELD_1_KEY, UNSTORED_1_FIELD_TEXT,
@@ -58,13 +69,58 @@
       Field.Store.NO, Field.Index.TOKENIZED, Field.TermVector.YES);
 
   public static Map nameValues = null;
-  
+
+  // ordered list of all the fields...
+  // could use LinkedHashMap for this purpose if Java1.4 is OK
+  public static Field[] fields = new Field[] {
+    textField1,
+    textField2,
+    textField3,
+    keyField,
+    noNormsField,
+    unIndField,
+    unStoredField1,
+    unStoredField2,
+  };
+
+  // Map<String fieldName, Field field>
+  public static Map all=new HashMap();
+  public static Map indexed=new HashMap();
+  public static Map stored=new HashMap();
+  public static Map unstored=new HashMap();
+  public static Map unindexed=new HashMap();
+  public static Map termvector=new HashMap();
+  public static Map notermvector=new HashMap();
+  public static Map noNorms=new HashMap();
+
+  static {
+    for (int i=0; i<fields.length; i++) {
+      Field f = fields[i];
+      add(all,f);
+      if (f.isIndexed()) add(indexed,f);
+      else add(unindexed,f);
+      if (f.isTermVectorStored()) add(termvector,f);
+      if (f.isIndexed() && !f.isTermVectorStored()) add(notermvector,f);
+      if (f.isStored()) add(stored,f);
+      else add(unstored,f);
+      if (f.getOmitNorms()) add(noNorms,f);
+    }
+  }
+
+
+  private static void add(Map map, Field field) {
+    map.put(field.name(), field);
+  }
+
+
   static
   {
     nameValues = new HashMap();
     nameValues.put(TEXT_FIELD_1_KEY, FIELD_1_TEXT);
     nameValues.put(TEXT_FIELD_2_KEY, FIELD_2_TEXT);
+    nameValues.put(TEXT_FIELD_3_KEY, FIELD_3_TEXT);
     nameValues.put(KEYWORD_FIELD_KEY, KEYWORD_TEXT);
+    nameValues.put(NO_NORMS_KEY, NO_NORMS_TEXT);
     nameValues.put(UNINDEXED_FIELD_KEY, UNINDEXED_FIELD_TEXT);
     nameValues.put(UNSTORED_FIELD_1_KEY, UNSTORED_1_FIELD_TEXT);
     nameValues.put(UNSTORED_FIELD_2_KEY, UNSTORED_2_FIELD_TEXT);
@@ -75,12 +131,9 @@
    * @param doc The document to write
    */ 
   public static void setupDoc(Document doc) {
-    doc.add(textField1);
-    doc.add(textField2);
-    doc.add(keyField);
-    doc.add(unIndField);
-    doc.add(unStoredField1);
-    doc.add(unStoredField2);
+    for (int i=0; i<fields.length; i++) {
+      doc.add(fields[i]);
+    }
   }                         
 
   /**

Modified: lucene/java/trunk/src/test/org/apache/lucene/index/TestDocumentWriter.java
URL: http://svn.apache.org/viewcvs/lucene/java/trunk/src/test/org/apache/lucene/index/TestDocumentWriter.java?rev=329526&r1=329525&r2=329526&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/index/TestDocumentWriter.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/index/TestDocumentWriter.java Sat Oct 29
22:42:09 2005
@@ -75,6 +75,15 @@
       fields = doc.getFields("keyField");
       assertTrue(fields != null && fields.length == 1);
       assertTrue(fields[0].stringValue().equals(DocHelper.KEYWORD_TEXT));
+
+      fields = doc.getFields(DocHelper.NO_NORMS_KEY);
+      assertTrue(fields != null && fields.length == 1);
+      assertTrue(fields[0].stringValue().equals(DocHelper.NO_NORMS_TEXT));
+
+      fields = doc.getFields(DocHelper.TEXT_FIELD_3_KEY);
+      assertTrue(fields != null && fields.length == 1);
+      assertTrue(fields[0].stringValue().equals(DocHelper.FIELD_3_TEXT));
+
     } catch (IOException e) {
       e.printStackTrace();
       assertTrue(false);

Modified: lucene/java/trunk/src/test/org/apache/lucene/index/TestFieldInfos.java
URL: http://svn.apache.org/viewcvs/lucene/java/trunk/src/test/org/apache/lucene/index/TestFieldInfos.java?rev=329526&r1=329525&r2=329526&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/index/TestFieldInfos.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/index/TestFieldInfos.java Sat Oct 29 22:42:09
2005
@@ -31,7 +31,7 @@
     FieldInfos fieldInfos = new FieldInfos();
     fieldInfos.add(testDoc);
     //Since the complement is stored as well in the fields map
-    assertTrue(fieldInfos.size() == 6); //this is 6 b/c we are using the no-arg constructor
+    assertTrue(fieldInfos.size() == DocHelper.all.size()); //this is all b/c we are using
the no-arg constructor
     RAMDirectory dir = new RAMDirectory();
     String name = "testFile";
     IndexOutput output = dir.createOutput(name);
@@ -47,11 +47,23 @@
       FieldInfo info = readIn.fieldInfo("textField1");
       assertTrue(info != null);
       assertTrue(info.storeTermVector == false);
-      
+      assertTrue(info.omitNorms == false);
+
       info = readIn.fieldInfo("textField2");
       assertTrue(info != null);
       assertTrue(info.storeTermVector == true);
-      
+      assertTrue(info.omitNorms == false);
+
+      info = readIn.fieldInfo("textField3");
+      assertTrue(info != null);
+      assertTrue(info.storeTermVector == false);
+      assertTrue(info.omitNorms == true);
+
+      info = readIn.fieldInfo("omitNorms");
+      assertTrue(info != null);
+      assertTrue(info.storeTermVector == false);
+      assertTrue(info.omitNorms == true);
+
       dir.close();
 
     } catch (IOException e) {

Modified: lucene/java/trunk/src/test/org/apache/lucene/index/TestFieldsReader.java
URL: http://svn.apache.org/viewcvs/lucene/java/trunk/src/test/org/apache/lucene/index/TestFieldsReader.java?rev=329526&r1=329525&r2=329526&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/index/TestFieldsReader.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/index/TestFieldsReader.java Sat Oct 29 22:42:09
2005
@@ -54,14 +54,22 @@
     assertTrue(doc != null);
     assertTrue(doc.getField("textField1") != null);
     
-    
     Field field = doc.getField("textField2");
     assertTrue(field != null);
     assertTrue(field.isTermVectorStored() == true);
     
     assertTrue(field.isStoreOffsetWithTermVector() == true);
     assertTrue(field.isStorePositionWithTermVector() == true);
-    
+    assertTrue(field.getOmitNorms() == false);
+
+    field = doc.getField("textField3");
+    assertTrue(field != null);
+    assertTrue(field.isTermVectorStored() == false);
+    assertTrue(field.isStoreOffsetWithTermVector() == false);
+    assertTrue(field.isStorePositionWithTermVector() == false);
+    assertTrue(field.getOmitNorms() == true);
+
+
     reader.close();
   }
 }

Modified: lucene/java/trunk/src/test/org/apache/lucene/index/TestMultiReader.java
URL: http://svn.apache.org/viewcvs/lucene/java/trunk/src/test/org/apache/lucene/index/TestMultiReader.java?rev=329526&r1=329525&r2=329526&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/index/TestMultiReader.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/index/TestMultiReader.java Sat Oct 29 22:42:09
2005
@@ -61,12 +61,13 @@
     assertTrue(reader != null);
     Document newDoc1 = reader.document(0);
     assertTrue(newDoc1 != null);
-    assertTrue(DocHelper.numFields(newDoc1) == DocHelper.numFields(doc1) - 2);
+    assertTrue(DocHelper.numFields(newDoc1) == DocHelper.numFields(doc1) - DocHelper.unstored.size());
     Document newDoc2 = reader.document(1);
     assertTrue(newDoc2 != null);
-    assertTrue(DocHelper.numFields(newDoc2) == DocHelper.numFields(doc2) - 2);
+    assertTrue(DocHelper.numFields(newDoc2) == DocHelper.numFields(doc2) - DocHelper.unstored.size());
     TermFreqVector vector = reader.getTermFreqVector(0, DocHelper.TEXT_FIELD_2_KEY);
     assertTrue(vector != null);
+    TestSegmentReader.checkNorms(reader);
   }
   
   public void testTermVectors() {

Modified: lucene/java/trunk/src/test/org/apache/lucene/index/TestSegmentMerger.java
URL: http://svn.apache.org/viewcvs/lucene/java/trunk/src/test/org/apache/lucene/index/TestSegmentMerger.java?rev=329526&r1=329525&r2=329526&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/index/TestSegmentMerger.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/index/TestSegmentMerger.java Sat Oct 29 22:42:09
2005
@@ -75,10 +75,10 @@
     Document newDoc1 = mergedReader.document(0);
     assertTrue(newDoc1 != null);
     //There are 2 unstored fields on the document
-    assertTrue(DocHelper.numFields(newDoc1) == DocHelper.numFields(doc1) - 2);
+    assertTrue(DocHelper.numFields(newDoc1) == DocHelper.numFields(doc1) - DocHelper.unstored.size());
     Document newDoc2 = mergedReader.document(1);
     assertTrue(newDoc2 != null);
-    assertTrue(DocHelper.numFields(newDoc2) == DocHelper.numFields(doc2) - 2);
+    assertTrue(DocHelper.numFields(newDoc2) == DocHelper.numFields(doc2) - DocHelper.unstored.size());
     
     TermDocs termDocs = mergedReader.termDocs(new Term(DocHelper.TEXT_FIELD_2_KEY, "field"));
     assertTrue(termDocs != null);
@@ -106,6 +106,8 @@
       //System.out.println("Term: " + term + " Freq: " + freq);
       assertTrue(DocHelper.FIELD_2_TEXT.indexOf(term) != -1);
       assertTrue(DocHelper.FIELD_2_FREQS[i] == freq);
-    }                                                
+    }
+
+    TestSegmentReader.checkNorms(mergedReader);
   }    
 }

Modified: lucene/java/trunk/src/test/org/apache/lucene/index/TestSegmentReader.java
URL: http://svn.apache.org/viewcvs/lucene/java/trunk/src/test/org/apache/lucene/index/TestSegmentReader.java?rev=329526&r1=329525&r2=329526&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/index/TestSegmentReader.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/index/TestSegmentReader.java Sat Oct 29 22:42:09
2005
@@ -20,6 +20,7 @@
 import org.apache.lucene.store.RAMDirectory;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
+import org.apache.lucene.search.DefaultSimilarity;
 
 import java.io.IOException;
 import java.util.Collection;
@@ -50,7 +51,7 @@
     assertTrue(dir != null);
     assertTrue(reader != null);
     assertTrue(DocHelper.nameValues.size() > 0);
-    assertTrue(DocHelper.numFields(testDoc) == 6);
+    assertTrue(DocHelper.numFields(testDoc) == DocHelper.all.size());
   }
   
   public void testDocument() throws IOException {
@@ -59,7 +60,7 @@
     Document result = reader.document(0);
     assertTrue(result != null);
     //There are 2 unstored fields on the document that are not preserved across writing
-    assertTrue(DocHelper.numFields(result) == DocHelper.numFields(testDoc) - 2);
+    assertTrue(DocHelper.numFields(result) == DocHelper.numFields(testDoc) - DocHelper.unstored.size());
     
     Enumeration fields = result.fields();
     while (fields.hasMoreElements()) {
@@ -91,7 +92,7 @@
   public void testGetFieldNameVariations() {
     Collection result = reader.getFieldNames(IndexReader.FieldOption.ALL);
     assertTrue(result != null);
-    assertTrue(result.size() == 6);
+    assertTrue(result.size() == DocHelper.all.size());
     for (Iterator iter = result.iterator(); iter.hasNext();) {
       String s = (String) iter.next();
       //System.out.println("Name: " + s);
@@ -99,23 +100,23 @@
     }                                                                               
     result = reader.getFieldNames(IndexReader.FieldOption.INDEXED);
     assertTrue(result != null);
-    assertTrue(result.size() == 5);
+    assertTrue(result.size() == DocHelper.indexed.size());
     for (Iterator iter = result.iterator(); iter.hasNext();) {
       String s = (String) iter.next();
-      assertTrue(DocHelper.nameValues.containsKey(s) == true || s.equals(""));
+      assertTrue(DocHelper.indexed.containsKey(s) == true || s.equals(""));
     }
     
     result = reader.getFieldNames(IndexReader.FieldOption.UNINDEXED);
     assertTrue(result != null);
-    assertTrue(result.size() == 1);
+    assertTrue(result.size() == DocHelper.unindexed.size());
     //Get all indexed fields that are storing term vectors
     result = reader.getFieldNames(IndexReader.FieldOption.INDEXED_WITH_TERMVECTOR);
     assertTrue(result != null);
-    assertTrue(result.size() == 2);
+    assertTrue(result.size() == DocHelper.termvector.size());
     
     result = reader.getFieldNames(IndexReader.FieldOption.INDEXED_NO_TERMVECTOR);
     assertTrue(result != null);
-    assertTrue(result.size() == 3);
+    assertTrue(result.size() == DocHelper.notermvector.size());
   } 
   
   public void testTerms() throws IOException {
@@ -134,6 +135,10 @@
     assertTrue(termDocs != null);
     termDocs.seek(new Term(DocHelper.TEXT_FIELD_1_KEY, "field"));
     assertTrue(termDocs.next() == true);
+
+    termDocs.seek(new Term(DocHelper.NO_NORMS_KEY,  DocHelper.NO_NORMS_TEXT));
+    assertTrue(termDocs.next() == true);
+
     
     TermPositions positions = reader.termPositions();
     positions.seek(new Term(DocHelper.TEXT_FIELD_1_KEY, "field"));
@@ -142,7 +147,7 @@
     assertTrue(positions.nextPosition() >= 0);
   }    
   
-  public void testNorms() {
+  public void testNorms() throws IOException {
     //TODO: Not sure how these work/should be tested
 /*
     try {
@@ -155,7 +160,32 @@
     }
 */
 
-  }    
+    checkNorms(reader);
+  }
+
+  public static void checkNorms(IndexReader reader) throws IOException {
+        // test omit norms
+    for (int i=0; i<DocHelper.fields.length; i++) {
+      Field f = DocHelper.fields[i];
+      if (f.isIndexed()) {
+        assertEquals(reader.hasNorms(f.name()), !f.getOmitNorms());
+        assertEquals(reader.hasNorms(f.name()), !DocHelper.noNorms.containsKey(f.name()));
+        if (!reader.hasNorms(f.name())) {
+          // test for fake norms of 1.0
+          byte [] norms = reader.norms(f.name());
+          assertEquals(norms.length,reader.maxDoc());
+          for (int j=0; j<reader.maxDoc(); j++) {
+            assertEquals(norms[j], DefaultSimilarity.encodeNorm(1.0f));
+          }
+          norms = new byte[reader.maxDoc()];
+          reader.norms(f.name(),norms, 0);
+          for (int j=0; j<reader.maxDoc(); j++) {
+            assertEquals(norms[j], DefaultSimilarity.encodeNorm(1.0f));
+          }
+        }
+      }
+    }
+  }
   
   public void testTermVectors() throws IOException {
     TermFreqVector result = reader.getTermFreqVector(0, DocHelper.TEXT_FIELD_2_KEY);



Mime
View raw message