lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From rm...@apache.org
Subject svn commit: r1055238 - in /lucene/dev/trunk/lucene: ./ contrib/demo/src/java/org/apache/lucene/demo/ contrib/instantiated/src/java/org/apache/lucene/store/instantiated/ contrib/instantiated/src/test/org/apache/lucene/store/instantiated/ contrib/misc/sr...
Date Tue, 04 Jan 2011 23:11:38 GMT
Author: rmuir
Date: Tue Jan  4 23:11:37 2011
New Revision: 1055238

URL: http://svn.apache.org/viewvc?rev=1055238&view=rev
Log:
LUCENE-2771: remove norms() from non-atomic IndexReaders

Modified:
    lucene/dev/trunk/lucene/CHANGES.txt
    lucene/dev/trunk/lucene/contrib/demo/src/java/org/apache/lucene/demo/SearchFiles.java
    lucene/dev/trunk/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndex.java
    lucene/dev/trunk/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestEmptyIndex.java
    lucene/dev/trunk/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java
    lucene/dev/trunk/lucene/contrib/misc/src/test/org/apache/lucene/index/TestFieldNormModifier.java
    lucene/dev/trunk/lucene/contrib/misc/src/test/org/apache/lucene/misc/TestLengthNormModifier.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DirectoryReader.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/MultiReader.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/ParallelReader.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/SlowMultiReaderWrapper.java
    lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexReader.java
    lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexReaderClone.java
    lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexReaderCloneNorms.java
    lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestNorms.java
    lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestSegmentReader.java

Modified: lucene/dev/trunk/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/CHANGES.txt?rev=1055238&r1=1055237&r2=1055238&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/CHANGES.txt (original)
+++ lucene/dev/trunk/lucene/CHANGES.txt Tue Jan  4 23:11:37 2011
@@ -119,6 +119,10 @@ Changes in backwards compatibility polic
   need to change it (e.g. using "\\" to escape '\' itself).  
   (Sunil Kamath, Terry Yang via Robert Muir)
 
+* LUCENE-2771: IndexReader.norms() now throws UOE on non-atomic IndexReaders. If
+  you really want a top-level norms, use MultiNorms or SlowMultiReaderWrapper.
+  (Uwe Schindler, Robert Muir)
+
 Changes in Runtime Behavior
 
 * LUCENE-2650, LUCENE-2825: The behavior of FSDirectory.open has changed. On 64-bit

Modified: lucene/dev/trunk/lucene/contrib/demo/src/java/org/apache/lucene/demo/SearchFiles.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/demo/src/java/org/apache/lucene/demo/SearchFiles.java?rev=1055238&r1=1055237&r2=1055238&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/demo/src/java/org/apache/lucene/demo/SearchFiles.java
(original)
+++ lucene/dev/trunk/lucene/contrib/demo/src/java/org/apache/lucene/demo/SearchFiles.java
Tue Jan  4 23:11:37 2011
@@ -27,7 +27,6 @@ import java.util.Date;
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.standard.StandardAnalyzer;
 import org.apache.lucene.document.Document;
-import org.apache.lucene.index.FilterIndexReader;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.queryParser.QueryParser;
 import org.apache.lucene.search.Collector;
@@ -43,31 +42,12 @@ import org.apache.lucene.util.Version;
 /** Simple command-line based search demo. */
 public class SearchFiles {
 
-  /** Use the norms from one field for all fields.  Norms are read into memory,
-   * using a byte of memory per document per searched field.  This can cause
-   * search of large collections with a large number of fields to run out of
-   * memory.  If all of the fields contain only a single token, then the norms
-   * are all identical, then single norm vector may be shared. */
-  private static class OneNormsReader extends FilterIndexReader {
-    private String field;
-
-    public OneNormsReader(IndexReader in, String field) {
-      super(in);
-      this.field = field;
-    }
-
-    @Override
-    public byte[] norms(String field) throws IOException {
-      return in.norms(this.field);
-    }
-  }
-
   private SearchFiles() {}
 
   /** Simple command-line based search demo. */
   public static void main(String[] args) throws Exception {
     String usage =
-      "Usage:\tjava org.apache.lucene.demo.SearchFiles [-index dir] [-field f] [-repeat n]
[-queries file] [-raw] [-norms field] [-paging hitsPerPage]";
+      "Usage:\tjava org.apache.lucene.demo.SearchFiles [-index dir] [-field f] [-repeat n]
[-queries file] [-raw] [-paging hitsPerPage]";
     usage += "\n\tSpecify 'false' for hitsPerPage to use streaming instead of paging search.";
     if (args.length > 0 && ("-h".equals(args[0]) || "-help".equals(args[0])))
{
       System.out.println(usage);
@@ -79,7 +59,6 @@ public class SearchFiles {
     String queries = null;
     int repeat = 0;
     boolean raw = false;
-    String normsField = null;
     boolean paging = true;
     int hitsPerPage = 10;
     
@@ -98,9 +77,6 @@ public class SearchFiles {
         i++;
       } else if ("-raw".equals(args[i])) {
         raw = true;
-      } else if ("-norms".equals(args[i])) {
-        normsField = args[i+1];
-        i++;
       } else if ("-paging".equals(args[i])) {
         if (args[i+1].equals("false")) {
           paging = false;
@@ -116,9 +92,6 @@ public class SearchFiles {
     
     IndexReader reader = IndexReader.open(FSDirectory.open(new File(index)), true); // only
searching, so read-only=true
 
-    if (normsField != null)
-      reader = new OneNormsReader(reader, normsField);
-
     Searcher searcher = new IndexSearcher(reader);
     Analyzer analyzer = new StandardAnalyzer(Version.LUCENE_CURRENT);
 

Modified: lucene/dev/trunk/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndex.java?rev=1055238&r1=1055237&r2=1055238&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndex.java
(original)
+++ lucene/dev/trunk/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndex.java
Tue Jan  4 23:11:37 2011
@@ -30,6 +30,7 @@ import org.apache.lucene.analysis.Analyz
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Fieldable;
 import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.MultiNorms;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.index.TermsEnum;
 import org.apache.lucene.index.Fields;
@@ -211,12 +212,11 @@ public class InstantiatedIndex
       }
     }
 
-
-
     // create norms
     for (String fieldName : allFieldNames) {
       if (fields == null || fields.contains(fieldName)) {
-        getNormsByFieldNameAndDocumentNumber().put(fieldName, sourceIndexReader.norms(fieldName));
+        byte norms[] = MultiNorms.norms(sourceIndexReader, fieldName);
+        getNormsByFieldNameAndDocumentNumber().put(fieldName, norms);
       }
     }
 

Modified: lucene/dev/trunk/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestEmptyIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestEmptyIndex.java?rev=1055238&r1=1055237&r2=1055238&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestEmptyIndex.java
(original)
+++ lucene/dev/trunk/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestEmptyIndex.java
Tue Jan  4 23:11:37 2011
@@ -22,6 +22,7 @@ import org.apache.lucene.analysis.MockAn
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.MultiFields;
+import org.apache.lucene.index.MultiNorms;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.TermQuery;
@@ -67,8 +68,7 @@ public class TestEmptyIndex extends Luce
   }
 
   private void testNorms(IndexReader r) throws IOException {
-    byte[] norms;
-    norms = r.norms("foo");
+    byte[] norms = MultiNorms.norms(r, "foo");
     if (norms != null) {
       assertEquals(0, norms.length);
       norms = new byte[10];

Modified: lucene/dev/trunk/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java?rev=1055238&r1=1055237&r2=1055238&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java
(original)
+++ lucene/dev/trunk/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java
Tue Jan  4 23:11:37 2011
@@ -30,6 +30,7 @@ import org.apache.lucene.document.Docume
 import org.apache.lucene.document.Field;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.MultiNorms;
 import org.apache.lucene.index.Payload;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.index.DocsEnum;
@@ -341,8 +342,8 @@ public class TestIndicesEquals extends L
 
       // test norms as used by normal use
 
-      byte[] aprioriNorms = aprioriReader.norms((String) field);
-      byte[] testNorms = testReader.norms((String) field);
+      byte[] aprioriNorms = MultiNorms.norms(aprioriReader, (String) field);
+      byte[] testNorms = MultiNorms.norms(testReader, (String) field);
 
       if (aprioriNorms != null) {
         assertEquals(aprioriNorms.length, testNorms.length);
@@ -354,10 +355,10 @@ public class TestIndicesEquals extends L
         // test norms as used by multireader
 
         aprioriNorms = new byte[aprioriReader.maxDoc()];
-        aprioriReader.norms((String) field, aprioriNorms, 0);
+        MultiNorms.norms(aprioriReader, (String) field, aprioriNorms, 0);
 
         testNorms = new byte[testReader.maxDoc()];
-        testReader.norms((String) field, testNorms, 0);
+        MultiNorms.norms(testReader, (String) field, testNorms, 0);
 
         assertEquals(aprioriNorms.length, testNorms.length);
 
@@ -369,10 +370,10 @@ public class TestIndicesEquals extends L
         // test norms as used by multireader
 
         aprioriNorms = new byte[aprioriReader.maxDoc() + 10];
-        aprioriReader.norms((String) field, aprioriNorms, 10);
+        MultiNorms.norms(aprioriReader, (String) field, aprioriNorms, 10);
 
         testNorms = new byte[testReader.maxDoc() + 10];
-        testReader.norms((String) field, testNorms, 10);
+        MultiNorms.norms(testReader, (String) field, testNorms, 10);
 
         assertEquals(aprioriNorms.length, testNorms.length);
         

Modified: lucene/dev/trunk/lucene/contrib/misc/src/test/org/apache/lucene/index/TestFieldNormModifier.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/misc/src/test/org/apache/lucene/index/TestFieldNormModifier.java?rev=1055238&r1=1055237&r2=1055238&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/misc/src/test/org/apache/lucene/index/TestFieldNormModifier.java
(original)
+++ lucene/dev/trunk/lucene/contrib/misc/src/test/org/apache/lucene/index/TestFieldNormModifier.java
Tue Jan  4 23:11:37 2011
@@ -84,7 +84,7 @@ public class TestFieldNormModifier exten
   public void testFieldWithNoNorm() throws Exception {
     
     IndexReader r = IndexReader.open(store, false);
-    byte[] norms = r.norms("nonorm");
+    byte[] norms = MultiNorms.norms(r, "nonorm");
     
     // sanity check, norms should all be 1
     assertTrue("Whoops we have norms?", !r.hasNorms("nonorm"));
@@ -98,7 +98,7 @@ public class TestFieldNormModifier exten
     // nothing should have changed
     r = IndexReader.open(store, false);
     
-    norms = r.norms("nonorm");
+    norms = MultiNorms.norms(r, "nonorm");
     assertTrue("Whoops we have norms?", !r.hasNorms("nonorm"));
     assertNull(norms);
 
@@ -183,14 +183,14 @@ public class TestFieldNormModifier exten
   public void testNormKiller() throws IOException {
 
     IndexReader r = IndexReader.open(store, false);
-    byte[] oldNorms = r.norms("untokfield");    
+    byte[] oldNorms = MultiNorms.norms(r, "untokfield");    
     r.close();
     
     FieldNormModifier fnm = new FieldNormModifier(store, s);
     fnm.reSetNorms("untokfield");
 
     r = IndexReader.open(store, false);
-    byte[] newNorms = r.norms("untokfield");
+    byte[] newNorms = MultiNorms.norms(r, "untokfield");
     r.close();
     assertFalse(Arrays.equals(oldNorms, newNorms));    
 

Modified: lucene/dev/trunk/lucene/contrib/misc/src/test/org/apache/lucene/misc/TestLengthNormModifier.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/misc/src/test/org/apache/lucene/misc/TestLengthNormModifier.java?rev=1055238&r1=1055237&r2=1055238&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/misc/src/test/org/apache/lucene/misc/TestLengthNormModifier.java
(original)
+++ lucene/dev/trunk/lucene/contrib/misc/src/test/org/apache/lucene/misc/TestLengthNormModifier.java
Tue Jan  4 23:11:37 2011
@@ -25,6 +25,7 @@ import org.apache.lucene.document.Field;
 import org.apache.lucene.index.FieldNormModifier;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.MultiNorms;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.Collector;
 import org.apache.lucene.search.DefaultSimilarity;
@@ -94,7 +95,7 @@ public class TestLengthNormModifier exte
     public void testFieldWithNoNorm() throws Exception {
 
 	IndexReader r = IndexReader.open(store, false);
-	byte[] norms = r.norms("nonorm");
+	byte[] norms = MultiNorms.norms(r, "nonorm");
 
 	// sanity check, norms should all be 1
 	assertTrue("Whoops we have norms?", !r.hasNorms("nonorm"));
@@ -112,7 +113,7 @@ public class TestLengthNormModifier exte
 	// nothing should have changed
 	r = IndexReader.open(store, false);
 	
-	norms = r.norms("nonorm");
+	norms = MultiNorms.norms(r, "nonorm");
 	assertTrue("Whoops we have norms?", !r.hasNorms("nonorm"));
   assertNull(norms);
 

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DirectoryReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DirectoryReader.java?rev=1055238&r1=1055237&r2=1055238&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DirectoryReader.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DirectoryReader.java Tue Jan
 4 23:11:37 2011
@@ -20,7 +20,6 @@ package org.apache.lucene.index;
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
@@ -31,7 +30,6 @@ import java.util.Set;
 
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.FieldSelector;
-import org.apache.lucene.search.Similarity;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.Lock;
 import org.apache.lucene.store.LockObtainFailedException;
@@ -64,7 +62,6 @@ class DirectoryReader extends IndexReade
   private SegmentReader[] subReaders;
   private int[] starts;                           // 1st docno for each segment
   private final Map<SegmentReader,ReaderUtil.Slice> subReaderToSlice = new HashMap<SegmentReader,ReaderUtil.Slice>();
-  private Map<String,byte[]> normsCache = new HashMap<String,byte[]>();
   private int maxDoc = 0;
   private int numDocs = -1;
   private boolean hasDeletions = false;
@@ -186,7 +183,7 @@ class DirectoryReader extends IndexReade
 
   /** This constructor is only used for {@link #reopen()} */
   DirectoryReader(Directory directory, SegmentInfos infos, SegmentReader[] oldReaders, int[]
oldStarts,
-                  Map<String,byte[]> oldNormsCache, boolean readOnly, boolean doClone,
int termInfosIndexDivisor, CodecProvider codecs) throws IOException {
+                  boolean readOnly, boolean doClone, int termInfosIndexDivisor, CodecProvider
codecs) throws IOException {
     this.directory = directory;
     this.readOnly = readOnly;
     this.segmentInfos = infos;
@@ -274,38 +271,6 @@ class DirectoryReader extends IndexReade
     
     // initialize the readers to calculate maxDoc before we try to reuse the old normsCache
     initialize(newReaders);
-    
-    // try to copy unchanged norms from the old normsCache to the new one
-    if (oldNormsCache != null) {
-      for (Map.Entry<String,byte[]> entry: oldNormsCache.entrySet()) {
-        String field = entry.getKey();
-        if (!hasNorms(field)) {
-          continue;
-        }
-
-        byte[] oldBytes = entry.getValue();
-
-        byte[] bytes = new byte[maxDoc()];
-
-        for (int i = 0; i < subReaders.length; i++) {
-          Integer oldReaderIndex = segmentReaders.get(subReaders[i].getSegmentName());
-
-          // this SegmentReader was not re-opened, we can copy all of its norms 
-          if (oldReaderIndex != null &&
-               (oldReaders[oldReaderIndex.intValue()] == subReaders[i] 
-                 || oldReaders[oldReaderIndex.intValue()].norms.get(field) == subReaders[i].norms.get(field)))
{
-            // we don't have to synchronize here: either this constructor is called from
a SegmentReader,
-            // in which case no old norms cache is present, or it is called from MultiReader.reopen(),
-            // which is synchronized
-            System.arraycopy(oldBytes, oldStarts[oldReaderIndex.intValue()], bytes, starts[i],
starts[i+1] - starts[i]);
-          } else {
-            subReaders[i].norms(field, bytes, starts[i]);
-          }
-        }
-
-        normsCache.put(field, bytes);      // update cache
-      }
-    }
   }
 
   /** {@inheritDoc} */
@@ -497,7 +462,7 @@ class DirectoryReader extends IndexReade
 
   private synchronized DirectoryReader doReopen(SegmentInfos infos, boolean doClone, boolean
openReadOnly) throws CorruptIndexException, IOException {
     DirectoryReader reader;
-    reader = new DirectoryReader(directory, infos, subReaders, starts, normsCache, openReadOnly,
doClone, termInfosIndexDivisor, codecs);
+    reader = new DirectoryReader(directory, infos, subReaders, starts, openReadOnly, doClone,
termInfosIndexDivisor, codecs);
     return reader;
   }
 
@@ -637,41 +602,18 @@ class DirectoryReader extends IndexReade
   @Override
   public synchronized byte[] norms(String field) throws IOException {
     ensureOpen();
-    byte[] bytes = normsCache.get(field);
-    if (bytes != null)
-      return bytes;          // cache hit
-    if (!hasNorms(field))
-      return null;
-
-    bytes = new byte[maxDoc()];
-    for (int i = 0; i < subReaders.length; i++)
-      subReaders[i].norms(field, bytes, starts[i]);
-    normsCache.put(field, bytes);      // update cache
-    return bytes;
+    throw new UnsupportedOperationException("please use MultiNorms.norms, or wrap your IndexReader
with SlowMultiReaderWrapper, if you really need a top level norms");
   }
 
   @Override
   public synchronized void norms(String field, byte[] result, int offset)
     throws IOException {
-    ensureOpen();
-    byte[] bytes = normsCache.get(field);
-    if (bytes==null && !hasNorms(field)) {
-      Arrays.fill(result, offset, result.length, Similarity.getDefault().encodeNormValue(1.0f));
-    } else if (bytes != null) {                           // cache hit
-      System.arraycopy(bytes, 0, result, offset, maxDoc());
-    } else {
-      for (int i = 0; i < subReaders.length; i++) {      // read from segments
-        subReaders[i].norms(field, result, offset + starts[i]);
-      }
-    }
+    throw new UnsupportedOperationException("please use MultiNorms.norms, or wrap your IndexReader
with SlowMultiReaderWrapper, if you really need a top level norms");
   }
 
   @Override
   protected void doSetNorm(int n, String field, byte value)
     throws CorruptIndexException, IOException {
-    synchronized (normsCache) {
-      normsCache.remove(field);                         // clear cache      
-    }
     int i = readerIndex(n);                           // find segment num
     subReaders[i].setNorm(n-starts[i], field, value); // dispatch
   }
@@ -864,7 +806,6 @@ class DirectoryReader extends IndexReade
   @Override
   protected synchronized void doClose() throws IOException {
     IOException ioe = null;
-    normsCache = null;
     for (int i = 0; i < subReaders.length; i++) {
       // try to close each reader, even if an exception is thrown
       try {

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/MultiReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/MultiReader.java?rev=1055238&r1=1055237&r2=1055238&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/MultiReader.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/MultiReader.java Tue Jan  4 23:11:37
2011
@@ -18,14 +18,12 @@ package org.apache.lucene.index;
  */
 
 import java.io.IOException;
-import java.util.Arrays;
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.Map;
 
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.FieldSelector;
-import org.apache.lucene.search.Similarity;
 import org.apache.lucene.search.FieldCache; // not great (circular); used only to purge FieldCache
entry on close
 import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.BytesRef;
@@ -38,7 +36,6 @@ public class MultiReader extends IndexRe
   private int[] starts;                           // 1st docno for each segment
   private final Map<IndexReader,ReaderUtil.Slice> subReaderToSlice = new HashMap<IndexReader,ReaderUtil.Slice>();
   private boolean[] decrefOnClose;                // remember which subreaders to decRef
on close
-  private Map<String,byte[]> normsCache = new HashMap<String,byte[]>();
   private int maxDoc = 0;
   private int numDocs = -1;
   private boolean hasDeletions = false;
@@ -316,45 +313,18 @@ public class MultiReader extends IndexRe
   
   @Override
   public synchronized byte[] norms(String field) throws IOException {
-    ensureOpen();
-    byte[] bytes = normsCache.get(field);
-    if (bytes != null)
-      return bytes;          // cache hit
-    if (!hasNorms(field))
-      return null;
-
-    bytes = new byte[maxDoc()];
-    for (int i = 0; i < subReaders.length; i++)
-      subReaders[i].norms(field, bytes, starts[i]);
-    normsCache.put(field, bytes);      // update cache
-    return bytes;
+    throw new UnsupportedOperationException("please use MultiNorms.norms, or wrap your IndexReader
with SlowMultiReaderWrapper, if you really need a top level norms");
   }
 
   @Override
   public synchronized void norms(String field, byte[] result, int offset)
     throws IOException {
-    ensureOpen();
-    byte[] bytes = normsCache.get(field);
-    for (int i = 0; i < subReaders.length; i++)      // read from segments
-      subReaders[i].norms(field, result, offset + starts[i]);
-
-    if (bytes==null && !hasNorms(field)) {
-      Arrays.fill(result, offset, result.length, Similarity.getDefault().encodeNormValue(1.0f));
-    } else if (bytes != null) {                         // cache hit
-      System.arraycopy(bytes, 0, result, offset, maxDoc());
-    } else {
-      for (int i = 0; i < subReaders.length; i++) {     // read from segments
-        subReaders[i].norms(field, result, offset + starts[i]);
-      }
-    }
+    throw new UnsupportedOperationException("please use MultiNorms.norms, or wrap your IndexReader
with SlowMultiReaderWrapper, if you really need a top level norms");
   }
 
   @Override
   protected void doSetNorm(int n, String field, byte value)
     throws CorruptIndexException, IOException {
-    synchronized (normsCache) {
-      normsCache.remove(field);                         // clear cache
-    }
     int i = readerIndex(n);                           // find segment num
     subReaders[i].setNorm(n-starts[i], field, value); // dispatch
   }

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/ParallelReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/ParallelReader.java?rev=1055238&r1=1055237&r2=1055238&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/ParallelReader.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/ParallelReader.java Tue Jan 
4 23:11:37 2011
@@ -23,6 +23,7 @@ import org.apache.lucene.document.FieldS
 import org.apache.lucene.document.Fieldable;
 import org.apache.lucene.util.Bits;
 import org.apache.lucene.search.FieldCache; // not great (circular); used only to purge FieldCache
entry on close
+import org.apache.lucene.search.Similarity;
 import org.apache.lucene.util.BytesRef;
 
 import java.io.IOException;
@@ -53,7 +54,8 @@ public class ParallelReader extends Inde
   private SortedMap<String,IndexReader> fieldToReader = new TreeMap<String,IndexReader>();
   private Map<IndexReader,Collection<String>> readerToFields = new HashMap<IndexReader,Collection<String>>();
   private List<IndexReader> storedFieldReaders = new ArrayList<IndexReader>();
-
+  private Map<String,byte[]> normsCache = new HashMap<String,byte[]>();
+  
   private int maxDoc;
   private int numDocs;
   private boolean hasDeletions;
@@ -141,6 +143,9 @@ public class ParallelReader extends Inde
       reader.incRef();
     }
     decrefOnClose.add(Boolean.valueOf(incRefReaders));
+    synchronized(normsCache) {
+      normsCache.clear(); // TODO: don't need to clear this for all fields really?
+    }
   }
 
   private class ParallelFieldsEnum extends FieldsEnum {
@@ -278,6 +283,7 @@ public class ParallelReader extends Inde
 
     if (reopened) {
       List<Boolean> newDecrefOnClose = new ArrayList<Boolean>();
+      // TODO: maybe add a special reopen-ctor for norm-copying?
       ParallelReader pr = new ParallelReader();
       for (int i = 0; i < readers.size(); i++) {
         IndexReader oldReader = readers.get(i);
@@ -419,27 +425,51 @@ public class ParallelReader extends Inde
   }
 
   @Override
-  public byte[] norms(String field) throws IOException {
+  public synchronized byte[] norms(String field) throws IOException {
     ensureOpen();
     IndexReader reader = fieldToReader.get(field);
-    return reader==null ? null : reader.norms(field);
+
+    if (reader==null)
+      return null;
+    
+    byte[] bytes = normsCache.get(field);
+    if (bytes != null)
+      return bytes;
+    if (!hasNorms(field))
+      return null;
+
+    bytes = MultiNorms.norms(reader, field);
+    normsCache.put(field, bytes);
+    return bytes;
   }
 
   @Override
-  public void norms(String field, byte[] result, int offset)
+  public synchronized void norms(String field, byte[] result, int offset)
     throws IOException {
+    // TODO: maybe optimize
     ensureOpen();
     IndexReader reader = fieldToReader.get(field);
-    if (reader!=null)
-      reader.norms(field, result, offset);
+    if (reader==null)
+      return;
+    
+    byte[] norms = norms(field);
+    if (norms == null) {
+      Arrays.fill(result, offset, result.length, Similarity.getDefault().encodeNormValue(1.0f));
+    } else {
+      System.arraycopy(norms, 0, result, offset, maxDoc());
+    }
   }
 
   @Override
   protected void doSetNorm(int n, String field, byte value)
     throws CorruptIndexException, IOException {
     IndexReader reader = fieldToReader.get(field);
-    if (reader!=null)
+    if (reader!=null) {
+      synchronized(normsCache) {
+        normsCache.remove(field);
+      }
       reader.doSetNorm(n, field, value);
+    }
   }
 
   @Override

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/SlowMultiReaderWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/SlowMultiReaderWrapper.java?rev=1055238&r1=1055237&r2=1055238&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/SlowMultiReaderWrapper.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/SlowMultiReaderWrapper.java Tue
Jan  4 23:11:37 2011
@@ -18,6 +18,13 @@ package org.apache.lucene.index;
  */
 
 import java.io.IOException;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Map;
+
+import org.apache.lucene.search.Similarity;
 import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.ReaderUtil; // javadoc
 
@@ -48,6 +55,8 @@ import org.apache.lucene.index.MultiRead
 
 public final class SlowMultiReaderWrapper extends FilterIndexReader {
 
+  private final Map<String,byte[]> normsCache = new HashMap<String,byte[]>();
+  
   public SlowMultiReaderWrapper(IndexReader other) {
     super(other);
   }
@@ -62,9 +71,44 @@ public final class SlowMultiReaderWrappe
     return MultiFields.getDeletedDocs(in);
   }
 
+  
   @Override
   public IndexReader[] getSequentialSubReaders() {
     return null;
   }
+
+  @Override
+  public synchronized byte[] norms(String field) throws IOException {
+    ensureOpen();
+    byte[] bytes = normsCache.get(field);
+    if (bytes != null)
+      return bytes;
+    if (!hasNorms(field))
+      return null;
+
+    bytes = MultiNorms.norms(in, field);
+    normsCache.put(field, bytes);
+    return bytes;
+  }
+
+  @Override
+  public synchronized void norms(String field, byte[] bytes, int offset) throws IOException
{
+    // TODO: maybe optimize
+    ensureOpen();
+    byte[] norms = norms(field);
+    if (norms == null) {
+      Arrays.fill(bytes, offset, bytes.length, Similarity.getDefault().encodeNormValue(1.0f));
+    } else {
+      System.arraycopy(norms, 0, bytes, offset, maxDoc());
+    }
+  }
   
+  @Override
+  protected void doSetNorm(int n, String field, byte value)
+      throws CorruptIndexException, IOException {
+    synchronized(normsCache) {
+      normsCache.remove(field);
+    }
+    in.doSetNorm(n, field, value);
+  }
 }

Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexReader.java?rev=1055238&r1=1055237&r2=1055238&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexReader.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexReader.java Tue Jan
 4 23:11:37 2011
@@ -1333,8 +1333,8 @@ public class TestIndexReader extends Luc
       it1 = fields1.iterator();
       while (it1.hasNext()) {
         String curField = it1.next();
-        byte[] norms1 = index1.norms(curField);
-        byte[] norms2 = index2.norms(curField);
+        byte[] norms1 = MultiNorms.norms(index1, curField);
+        byte[] norms2 = MultiNorms.norms(index2, curField);
         if (norms1 != null && norms2 != null)
         {
           assertEquals(norms1.length, norms2.length);

Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexReaderClone.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexReaderClone.java?rev=1055238&r1=1055237&r2=1055238&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexReaderClone.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexReaderClone.java Tue
Jan  4 23:11:37 2011
@@ -272,13 +272,13 @@ public class TestIndexReaderClone extend
    * @throws Exception
    */
   private void performDefaultTests(IndexReader r1) throws Exception {
-    float norm1 = Similarity.getDefault().decodeNormValue(r1.norms("field1")[4]);
+    float norm1 = Similarity.getDefault().decodeNormValue(MultiNorms.norms(r1, "field1")[4]);
 
     IndexReader pr1Clone = (IndexReader) r1.clone();
     pr1Clone.deleteDocument(10);
     pr1Clone.setNorm(4, "field1", 0.5f);
-    assertTrue(Similarity.getDefault().decodeNormValue(r1.norms("field1")[4]) == norm1);
-    assertTrue(Similarity.getDefault().decodeNormValue(pr1Clone.norms("field1")[4]) != norm1);
+    assertTrue(Similarity.getDefault().decodeNormValue(MultiNorms.norms(r1, "field1")[4])
== norm1);
+    assertTrue(Similarity.getDefault().decodeNormValue(MultiNorms.norms(pr1Clone, "field1")[4])
!= norm1);
 
     final Bits delDocs = MultiFields.getDeletedDocs(r1);
     assertTrue(delDocs == null || !delDocs.get(10));
@@ -428,7 +428,7 @@ public class TestIndexReaderClone extend
     IndexReader orig = IndexReader.open(dir1, false);
     orig.setNorm(1, "field1", 17.0f);
     final byte encoded = Similarity.getDefault().encodeNormValue(17.0f);
-    assertEquals(encoded, orig.norms("field1")[1]);
+    assertEquals(encoded, MultiNorms.norms(orig, "field1")[1]);
 
     // the cloned segmentreader should have 2 references, 1 to itself, and 1 to
     // the original segmentreader
@@ -437,7 +437,7 @@ public class TestIndexReaderClone extend
     clonedReader.close();
 
     IndexReader r = IndexReader.open(dir1, false);
-    assertEquals(encoded, r.norms("field1")[1]);
+    assertEquals(encoded, MultiNorms.norms(r, "field1")[1]);
     r.close();
     dir1.close();
   }

Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexReaderCloneNorms.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexReaderCloneNorms.java?rev=1055238&r1=1055237&r2=1055238&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexReaderCloneNorms.java
(original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexReaderCloneNorms.java
Tue Jan  4 23:11:37 2011
@@ -272,7 +272,7 @@ public class TestIndexReaderCloneNorms e
   private void verifyIndex(IndexReader ir) throws IOException {
     for (int i = 0; i < NUM_FIELDS; i++) {
       String field = "f" + i;
-      byte b[] = ir.norms(field);
+      byte b[] = MultiNorms.norms(ir, field);
       assertEquals("number of norms mismatches", numDocNorms, b.length);
       ArrayList<Float> storedNorms = (i == 1 ? modifiedNorms : norms);
       for (int j = 0; j < b.length; j++) {

Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestNorms.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestNorms.java?rev=1055238&r1=1055237&r2=1055238&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestNorms.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestNorms.java Tue Jan  4 23:11:37
2011
@@ -179,7 +179,7 @@ public class TestNorms extends LuceneTes
     IndexReader ir = IndexReader.open(dir, false);
     for (int i = 0; i < NUM_FIELDS; i++) {
       String field = "f"+i;
-      byte b[] = ir.norms(field);
+      byte b[] = MultiNorms.norms(ir, field);
       assertEquals("number of norms mismatches",numDocNorms,b.length);
       ArrayList<Float> storedNorms = (i==1 ? modifiedNorms : norms);
       for (int j = 0; j < b.length; j++) {

Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestSegmentReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestSegmentReader.java?rev=1055238&r1=1055237&r2=1055238&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestSegmentReader.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestSegmentReader.java Tue Jan
 4 23:11:37 2011
@@ -181,11 +181,11 @@ public class TestSegmentReader extends L
         assertEquals(reader.hasNorms(f.name()), !DocHelper.noNorms.containsKey(f.name()));
         if (!reader.hasNorms(f.name())) {
           // test for fake norms of 1.0 or null depending on the flag
-          byte [] norms = reader.norms(f.name());
+          byte [] norms = MultiNorms.norms(reader, f.name());
           byte norm1 = Similarity.getDefault().encodeNormValue(1.0f);
           assertNull(norms);
           norms = new byte[reader.maxDoc()];
-          reader.norms(f.name(),norms, 0);
+          MultiNorms.norms(reader, f.name(),norms, 0);
           for (int j=0; j<reader.maxDoc(); j++) {
             assertEquals(norms[j], norm1);
           }



Mime
View raw message