lucene-java-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hoss...@apache.org
Subject svn commit: r803676 [2/2] - in /lucene/java/trunk: ./ contrib/remote/src/test/org/apache/lucene/search/ src/java/org/apache/lucene/search/ src/java/org/apache/lucene/util/ src/test/org/apache/lucene/search/ src/test/org/apache/lucene/search/function/ s...
Date Wed, 12 Aug 2009 19:31:39 GMT
Modified: lucene/java/trunk/src/test/org/apache/lucene/search/function/TestOrdValues.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/search/function/TestOrdValues.java?rev=803676&r1=803675&r2=803676&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/search/function/TestOrdValues.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/search/function/TestOrdValues.java Wed Aug
12 19:31:38 2009
@@ -18,6 +18,7 @@
  */
 
 import org.apache.lucene.index.CorruptIndexException;
+import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.QueryUtils;
@@ -150,11 +151,16 @@
       ScoreDoc[] h = s.search(q, null, 1000).scoreDocs;
       try {
         assertEquals("All docs should be matched!",N_DOCS,h.length);
-        if (i==0) {
-          innerArray = q.valSrc.getValues(s.getIndexReader()).getInnerArray();
-        } else {
-          log(i+".  compare: "+innerArray+" to "+q.valSrc.getValues(s.getIndexReader()).getInnerArray());
-          assertSame("field values should be cached and reused!", innerArray, q.valSrc.getValues(s.getIndexReader()).getInnerArray());
+        IndexReader[] readers = s.getIndexReader().getSequentialSubReaders();
+
+        for(int j = 0; j < readers.length; j++) {
+          IndexReader reader = readers[j];
+          if (i==0) {
+            innerArray = q.valSrc.getValues(reader).getInnerArray();
+          } else {
+            log(i+".  compare: "+innerArray+" to "+q.valSrc.getValues(reader).getInnerArray());
+            assertSame("field values should be cached and reused!", innerArray, q.valSrc.getValues(reader).getInnerArray());
+          }
         }
       } catch (UnsupportedOperationException e) {
         if (!warned) {
@@ -179,13 +185,22 @@
     q = new ValueSourceQuery(vs);
     h = s.search(q, null, 1000).scoreDocs;
     assertEquals("All docs should be matched!",N_DOCS,h.length);
-    try {
-      log("compare (should differ): "+innerArray+" to "+q.valSrc.getValues(s.getIndexReader()).getInnerArray());
-      assertNotSame("different values shuold be loaded for a different field!", innerArray,
q.valSrc.getValues(s.getIndexReader()).getInnerArray());
-    } catch (UnsupportedOperationException e) {
-      if (!warned) {
-        System.err.println("WARNING: "+testName()+" cannot fully test values of "+q);
-        warned = true;
+    IndexReader[] readers = s.getIndexReader().getSequentialSubReaders();
+
+    for (int j = 0; j < readers.length; j++) {
+      IndexReader reader = readers[j];
+      try {
+        log("compare (should differ): " + innerArray + " to "
+            + q.valSrc.getValues(reader).getInnerArray());
+        assertNotSame(
+            "different values shuold be loaded for a different field!",
+            innerArray, q.valSrc.getValues(reader).getInnerArray());
+      } catch (UnsupportedOperationException e) {
+        if (!warned) {
+          System.err.println("WARNING: " + testName()
+              + " cannot fully test values of " + q);
+          warned = true;
+        }
       }
     }
 
@@ -199,13 +214,22 @@
     q = new ValueSourceQuery(vs);
     h = s.search(q, null, 1000).scoreDocs;
     assertEquals("All docs should be matched!",N_DOCS,h.length);
-    try {
-      log("compare (should differ): "+innerArray+" to "+q.valSrc.getValues(s.getIndexReader()).getInnerArray());
-      assertNotSame("cached field values should not be reused if reader as changed!", innerArray,
q.valSrc.getValues(s.getIndexReader()).getInnerArray());
-    } catch (UnsupportedOperationException e) {
-      if (!warned) {
-        System.err.println("WARNING: "+testName()+" cannot fully test values of "+q);
-        warned = true;
+    readers = s.getIndexReader().getSequentialSubReaders();
+
+    for (int j = 0; j < readers.length; j++) {
+      IndexReader reader = readers[j];
+      try {
+        log("compare (should differ): " + innerArray + " to "
+            + q.valSrc.getValues(reader).getInnerArray());
+        assertNotSame(
+            "cached field values should not be reused if reader as changed!",
+            innerArray, q.valSrc.getValues(reader).getInnerArray());
+      } catch (UnsupportedOperationException e) {
+        if (!warned) {
+          System.err.println("WARNING: " + testName()
+              + " cannot fully test values of " + q);
+          warned = true;
+        }
       }
     }
   }

Modified: lucene/java/trunk/src/test/org/apache/lucene/util/LuceneTestCase.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/util/LuceneTestCase.java?rev=803676&r1=803675&r2=803676&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/util/LuceneTestCase.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/util/LuceneTestCase.java Wed Aug 12 19:31:38
2009
@@ -17,21 +17,36 @@
  * limitations under the License.
  */
 
-import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.index.ConcurrentMergeScheduler;
-import junit.framework.TestCase;
+import java.io.PrintStream;
+import java.util.Arrays;
+import java.util.Iterator;
 import java.util.Random;
 
-/** Base class for all Lucene unit tests.  Currently the
- *  only added functionality over JUnit's TestCase is
- *  asserting that no unhandled exceptions occurred in
- *  threads launched by ConcurrentMergeScheduler.  If you
- *  override either <code>setUp()</code> or
- *  <code>tearDown()</code> in your unit test, make sure you
- *  call <code>super.setUp()</code> and
- *  <code>super.tearDown()</code>.
- */
+import junit.framework.TestCase;
 
+import org.apache.lucene.index.ConcurrentMergeScheduler;
+import org.apache.lucene.search.FieldCache;
+import org.apache.lucene.search.FieldCache.CacheEntry;
+import org.apache.lucene.util.FieldCacheSanityChecker.Insanity;
+
+/** 
+ * Base class for all Lucene unit tests.  
+ * <p>
+ * Currently the
+ * only added functionality over JUnit's TestCase is
+ * asserting that no unhandled exceptions occurred in
+ * threads launched by ConcurrentMergeScheduler and asserting sane
+ * FieldCache usage athe moment of tearDown.
+ * </p>
+ * <p>
+ * If you
+ * override either <code>setUp()</code> or
+ * <code>tearDown()</code> in your unit test, make sure you
+ * call <code>super.setUp()</code> and
+ * <code>super.tearDown()</code>
+ * </p>
+ * @see #assertSaneFieldCaches
+ */
 public abstract class LuceneTestCase extends TestCase {
 
   public LuceneTestCase() {
@@ -46,13 +61,112 @@
     ConcurrentMergeScheduler.setTestMode();
   }
 
+  /**
+   * Forcible purges all cache entries from the FieldCache.
+   * <p>
+   * This method will be called by tearDown to clean up FieldCache.DEFAULT.
+   * If a (poorly written) test has some expectation that the FieldCache
+   * will persist across test methods (ie: a static IndexReader) this 
+   * method can be overridden to do nothing.
+   * </p>
+   * @see FieldCache#purgeAllCaches()
+   */
+  protected void purgeFieldCache(final FieldCache fc) {
+    fc.purgeAllCaches();
+  }
+
+  protected String getTestLabel() {
+    return getClass().getName() + "." + getName();
+  }
+
   protected void tearDown() throws Exception {
-    if (ConcurrentMergeScheduler.anyUnhandledExceptions()) {
-      // Clear the failure so that we don't just keep
-      // failing subsequent test cases
-      ConcurrentMergeScheduler.clearUnhandledExceptions();
-      fail("ConcurrentMergeScheduler hit unhandled exceptions");
+    try {
+      // this isn't as useful as calling directly from the scope where the 
+      // index readers are used, because they could be gc'ed just before
+      // tearDown is called.
+      // But it's better then nothing.
+      assertSaneFieldCaches(getTestLabel());
+      
+      if (ConcurrentMergeScheduler.anyUnhandledExceptions()) {
+        // Clear the failure so that we don't just keep
+        // failing subsequent test cases
+        ConcurrentMergeScheduler.clearUnhandledExceptions();
+        fail("ConcurrentMergeScheduler hit unhandled exceptions");
+      }
+    } finally {
+      purgeFieldCache(FieldCache.DEFAULT);
+    }
+  }
+
+  /** 
+   * Asserts that FieldCacheSanityChecker does not detect any 
+   * problems with FieldCache.DEFAULT.
+   * <p>
+   * If any problems are found, they are logged to System.err 
+   * (allong with the msg) when the Assertion is thrown.
+   * </p>
+   * <p>
+   * This method is called by tearDown after every test method, 
+   * however IndexReaders scoped inside test methods may be garbage 
+   * collected prior to this method being called, causing errors to 
+   * be overlooked. Tests are encouraged to keep their IndexReaders 
+   * scoped at the class level, or to explicitly call this method 
+   * directly in the same scope as the IndexReader.
+   * </p>
+   * @see FieldCacheSanityChecker
+   */
+  protected void assertSaneFieldCaches(final String msg) {
+    final CacheEntry[] entries = FieldCache.DEFAULT.getCacheEntries();
+    Insanity[] insanity = null;
+    try {
+      try {
+        insanity = FieldCacheSanityChecker.checkSanity(entries);
+      } catch (RuntimeException e) {
+        dumpArray(msg+ ": FieldCache", entries, System.err);
+        throw e;
+      }
+
+      assertEquals(msg + ": Insane FieldCache usage(s) found", 
+                   0, insanity.length);
+      insanity = null;
+    } finally {
+
+      // report this in the event of any exception/failure
+      // if no failure, then insanity will be null anyway
+      if (null != insanity) {
+        dumpArray(msg + ": Insane FieldCache usage(s)", insanity, System.err);
+      }
+
+    }
+  }
+
+  /**
+   * Convinience method for logging an iterator.
+   * @param label String logged before/after the items in the iterator
+   * @param iter Each next() is toString()ed and logged on it's own line. If iter is null
this is logged differnetly then an empty iterator.
+   * @param stream Stream to log messages to.
+   */
+  public static void dumpIterator(String label, Iterator iter, 
+                                  PrintStream stream) {
+    stream.println("*** BEGIN "+label+" ***");
+    if (null == iter) {
+      stream.println(" ... NULL ...");
+    } else {
+      while (iter.hasNext()) {
+        stream.println(iter.next().toString());
+      }
     }
+    stream.println("*** END "+label+" ***");
+  }
+
+  /** 
+   * Convinience method for logging an array.  Wraps the array in an iterator and delegates
+   * @see dumpIterator(String,Iterator,PrintStream)
+   */
+  public static void dumpArray(String label, Object[] objs, 
+                               PrintStream stream) {
+    Iterator iter = (null == objs) ? null : Arrays.asList(objs).iterator();
+    dumpIterator(label, iter, stream);
   }
   
   /**

Added: lucene/java/trunk/src/test/org/apache/lucene/util/TestFieldCacheSanityChecker.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/util/TestFieldCacheSanityChecker.java?rev=803676&view=auto
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/util/TestFieldCacheSanityChecker.java (added)
+++ lucene/java/trunk/src/test/org/apache/lucene/util/TestFieldCacheSanityChecker.java Wed
Aug 12 19:31:38 2009
@@ -0,0 +1,181 @@
+package org.apache.lucene.util;
+
+/**
+ * Copyright 2009 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.analysis.WhitespaceAnalyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.search.FieldCache;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.MultiReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.store.RAMDirectory;
+import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.FieldCacheSanityChecker.Insanity;
+import org.apache.lucene.util.FieldCacheSanityChecker.InsanityType;
+
+import java.io.IOException;
+
+public class TestFieldCacheSanityChecker extends LuceneTestCase {
+
+  protected IndexReader readerA;
+  protected IndexReader readerB;
+  protected IndexReader readerX;
+
+  private static final int NUM_DOCS = 1000;
+
+  protected void setUp() throws Exception {
+    super.setUp();
+
+    RAMDirectory dirA = new RAMDirectory();
+    RAMDirectory dirB = new RAMDirectory();
+
+    IndexWriter wA = new IndexWriter(dirA, new WhitespaceAnalyzer(), true, 
+                                     IndexWriter.MaxFieldLength.LIMITED);
+    IndexWriter wB = new IndexWriter(dirB, new WhitespaceAnalyzer(), true, 
+                                     IndexWriter.MaxFieldLength.LIMITED);
+
+    long theLong = Long.MAX_VALUE;
+    double theDouble = Double.MAX_VALUE;
+    byte theByte = Byte.MAX_VALUE;
+    short theShort = Short.MAX_VALUE;
+    int theInt = Integer.MAX_VALUE;
+    float theFloat = Float.MAX_VALUE;
+    for (int i = 0; i < NUM_DOCS; i++){
+      Document doc = new Document();
+      doc.add(new Field("theLong", String.valueOf(theLong--), Field.Store.NO, Field.Index.NOT_ANALYZED));
+      doc.add(new Field("theDouble", String.valueOf(theDouble--), Field.Store.NO, Field.Index.NOT_ANALYZED));
+      doc.add(new Field("theByte", String.valueOf(theByte--), Field.Store.NO, Field.Index.NOT_ANALYZED));
+      doc.add(new Field("theShort", String.valueOf(theShort--), Field.Store.NO, Field.Index.NOT_ANALYZED));
+      doc.add(new Field("theInt", String.valueOf(theInt--), Field.Store.NO, Field.Index.NOT_ANALYZED));
+      doc.add(new Field("theFloat", String.valueOf(theFloat--), Field.Store.NO, Field.Index.NOT_ANALYZED));
+      if (0 == i % 3) {
+        wA.addDocument(doc);
+      } else {
+        wB.addDocument(doc);
+      }
+    }
+    wA.close();
+    wB.close();
+    readerA = IndexReader.open(dirA);
+    readerB = IndexReader.open(dirB);
+    readerX = new MultiReader(new IndexReader[] { readerA, readerB });
+  }
+
+  public void tearDown() throws Exception {
+    super.tearDown();
+    readerA.close();
+    readerB.close();
+    readerX.close();
+  }
+
+  public void testSanity() throws IOException {
+    FieldCache cache = FieldCache.DEFAULT;
+    cache.purgeAllCaches();
+
+    double [] doubles;
+    int [] ints;
+
+    doubles = cache.getDoubles(readerA, "theDouble");
+    doubles = cache.getDoubles(readerA, "theDouble", 
+                               FieldCache.DEFAULT_DOUBLE_PARSER);
+    doubles = cache.getDoubles(readerB, "theDouble", 
+                               FieldCache.DEFAULT_DOUBLE_PARSER);
+
+    ints = cache.getInts(readerX, "theInt");
+    ints = cache.getInts(readerX, "theInt", 
+                         FieldCache.DEFAULT_INT_PARSER);
+
+    // // // 
+
+    Insanity[] insanity = 
+      FieldCacheSanityChecker.checkSanity(cache.getCacheEntries());
+    
+    if (0 < insanity.length)
+      dumpArray(getTestLabel() + " INSANITY", insanity, System.err);
+
+    assertEquals("shouldn't be any cache insanity", 0, insanity.length);
+    cache.purgeAllCaches();
+  }
+
+  public void testInsanity1() throws IOException {
+    FieldCache cache = FieldCache.DEFAULT;
+    cache.purgeAllCaches();
+
+    int [] ints;
+    String [] strings;
+    byte [] bytes;
+
+    ints = cache.getInts(readerX, "theInt", FieldCache.DEFAULT_INT_PARSER);
+    strings = cache.getStrings(readerX, "theInt");
+
+    // this one is ok
+    bytes = cache.getBytes(readerX, "theByte");
+
+    // // // 
+
+    Insanity[] insanity = 
+      FieldCacheSanityChecker.checkSanity(cache.getCacheEntries());
+
+    assertEquals("wrong number of cache errors", 1, insanity.length);
+    assertEquals("wrong type of cache error", 
+                 InsanityType.VALUEMISMATCH,
+                 insanity[0].getType());
+    assertEquals("wrong number of entries in cache error", 2,
+                 insanity[0].getCacheEntries().length);
+
+    // we expect bad things, don't let tearDown complain about them
+    cache.purgeAllCaches();
+  }
+
+  public void testInsanity2() throws IOException {
+    FieldCache cache = FieldCache.DEFAULT;
+    cache.purgeAllCaches();
+
+    String [] strings;
+    byte [] bytes;
+
+    strings = cache.getStrings(readerA, "theString");
+    strings = cache.getStrings(readerB, "theString");
+    strings = cache.getStrings(readerX, "theString");
+
+    // this one is ok
+    bytes = cache.getBytes(readerX, "theByte");
+
+
+    // // // 
+
+    Insanity[] insanity = 
+      FieldCacheSanityChecker.checkSanity(cache.getCacheEntries());
+    
+    assertEquals("wrong number of cache errors", 1, insanity.length);
+    assertEquals("wrong type of cache error", 
+                 InsanityType.SUBREADER,
+                 insanity[0].getType());
+    assertEquals("wrong number of entries in cache error", 3,
+                 insanity[0].getCacheEntries().length);
+
+    // we expect bad things, don't let tearDown complain about them
+    cache.purgeAllCaches();
+  }
+  
+  public void testInsanity3() throws IOException {
+
+    // :TODO: subreader tree walking is really hairy ... add more crazy tests.
+  }
+
+}

Propchange: lucene/java/trunk/src/test/org/apache/lucene/util/TestFieldCacheSanityChecker.java
------------------------------------------------------------------------------
    svn:eol-style = native

Propchange: lucene/java/trunk/src/test/org/apache/lucene/util/TestFieldCacheSanityChecker.java
------------------------------------------------------------------------------
    svn:keywords = Date Author Id Revision HeadURL

Added: lucene/java/trunk/src/test/org/apache/lucene/util/TestRamUsageEstimator.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/util/TestRamUsageEstimator.java?rev=803676&view=auto
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/util/TestRamUsageEstimator.java (added)
+++ lucene/java/trunk/src/test/org/apache/lucene/util/TestRamUsageEstimator.java Wed Aug 12
19:31:38 2009
@@ -0,0 +1,57 @@
+package org.apache.lucene.util;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import junit.framework.TestCase;
+
+public class TestRamUsageEstimator extends TestCase {
+
+  public void testBasic() {
+    String string = new String("test str");
+    RamUsageEstimator rue = new RamUsageEstimator();
+    long size = rue.estimateRamUsage(string);
+    System.out.println("size:" + size);
+    
+    string = new String("test strin");
+    size = rue.estimateRamUsage(string);
+    System.out.println("size:" + size);
+    
+    Holder holder = new Holder();
+    holder.holder = new Holder("string2", 5000L);
+    size = rue.estimateRamUsage(holder);
+    System.out.println("size:" + size);
+    
+    String[] strings = new String[]{new String("test strin"), new String("hollow"), new String("catchmaster")};
+    size = rue.estimateRamUsage(strings);
+    System.out.println("size:" + size);
+  }
+  
+  private static final class Holder {
+    long field1 = 5000L;
+    String name = "name";
+    Holder holder;
+    
+    Holder() {
+    }
+    
+    Holder(String name, long field1) {
+      this.name = name;
+      this.field1 = field1;
+    }
+  }
+}

Propchange: lucene/java/trunk/src/test/org/apache/lucene/util/TestRamUsageEstimator.java
------------------------------------------------------------------------------
    svn:eol-style = native

Propchange: lucene/java/trunk/src/test/org/apache/lucene/util/TestRamUsageEstimator.java
------------------------------------------------------------------------------
    svn:keywords = Date Author Id Revision HeadURL



Mime
View raw message