lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mikemcc...@apache.org
Subject svn commit: r1416914 - in /lucene/dev/branches/lucene4547/lucene: codecs/src/java/org/apache/lucene/codecs/memory/ codecs/src/java/org/apache/lucene/codecs/simpletext/ core/src/java/org/apache/lucene/codecs/ core/src/java/org/apache/lucene/codecs/lucen...
Date Tue, 04 Dec 2012 12:37:09 GMT
Author: mikemccand
Date: Tue Dec  4 12:37:08 2012
New Revision: 1416914

URL: http://svn.apache.org/viewvc?rev=1416914&view=rev
Log:
make clone() explicit, move responsibility out of codec into SCR

Modified:
    lucene/dev/branches/lucene4547/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesFormat.java
    lucene/dev/branches/lucene4547/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSimpleDocValuesFormat.java
    lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/SimpleDVProducer.java
    lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41DocValuesProducer.java
    lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldDocValuesFormat.java
    lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/AtomicReader.java
    lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/SegmentCoreReaders.java
    lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/index/TestDocValuesIndexing.java
    lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/search/TestSameScoresWithThreads.java

Modified: lucene/dev/branches/lucene4547/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesFormat.java?rev=1416914&r1=1416913&r2=1416914&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesFormat.java
(original)
+++ lucene/dev/branches/lucene4547/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesFormat.java
Tue Dec  4 12:37:08 2012
@@ -132,6 +132,7 @@ public class MemoryDocValuesFormat exten
         };
       }
 
+      @Override
       public SortedDocValues getSorted(FieldInfo field) throws IOException {
         SortedDocValues valuesIn = producer.getSorted(field);
         final int maxDoc = valuesIn.size();
@@ -189,6 +190,12 @@ public class MemoryDocValuesFormat exten
       }
 
       @Override
+      public SimpleDVProducer clone() {
+        // We are already thread-safe:
+        return this;
+      }
+
+      @Override
       public void close() throws IOException {
         producer.close();
       }

Modified: lucene/dev/branches/lucene4547/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSimpleDocValuesFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSimpleDocValuesFormat.java?rev=1416914&r1=1416913&r2=1416914&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSimpleDocValuesFormat.java
(original)
+++ lucene/dev/branches/lucene4547/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSimpleDocValuesFormat.java
Tue Dec  4 12:37:08 2012
@@ -487,6 +487,17 @@ public class SimpleTextSimpleDocValuesFo
       assert !fields.isEmpty();
     }
 
+    private SimpleTextDocValuesReader(SimpleTextDocValuesReader other) {
+      this.data = other.data.clone();
+      this.fields.putAll(other.fields);
+      this.maxDoc = other.maxDoc;
+    }
+
+    @Override
+    public SimpleDVProducer clone() {
+      return new SimpleTextDocValuesReader(this);
+    }
+
     @Override
     public NumericDocValues getNumeric(FieldInfo fieldInfo) throws IOException {
       final OneField field = fields.get(fieldInfo.name);
@@ -505,7 +516,7 @@ public class SimpleTextSimpleDocValuesFo
       // valid:
       assert field != null: "field=" + fieldInfo.name + " fields=" + fields;
 
-      final IndexInput in = data.clone();
+      final IndexInput in = data;
       final BytesRef scratch = new BytesRef();
       final DecimalFormat decoder = new DecimalFormat(field.pattern, new DecimalFormatSymbols(Locale.ROOT));
 
@@ -515,6 +526,7 @@ public class SimpleTextSimpleDocValuesFo
         @Override
         public long get(int docID) {
           try {
+            //System.out.println(Thread.currentThread().getName() + ": get docID=" + docID
+ " in=" + in);
             if (docID < 0 || docID >= maxDoc) {
               throw new IndexOutOfBoundsException("docID must be 0 .. " + (maxDoc-1) + ";
got " + docID);
             }
@@ -560,7 +572,7 @@ public class SimpleTextSimpleDocValuesFo
       // valid:
       assert field != null;
 
-      final IndexInput in = data.clone();
+      final IndexInput in = data;
       final BytesRef scratch = new BytesRef();
       final DecimalFormat decoder = new DecimalFormat(field.pattern, new DecimalFormatSymbols(Locale.ROOT));
 
@@ -616,7 +628,7 @@ public class SimpleTextSimpleDocValuesFo
       // valid:
       assert field != null;
 
-      final IndexInput in = data.clone();
+      final IndexInput in = data;
       final BytesRef scratch = new BytesRef();
       final DecimalFormat decoder = new DecimalFormat(field.pattern, new DecimalFormatSymbols(Locale.ROOT));
       final DecimalFormat ordDecoder = new DecimalFormat(field.ordPattern, new DecimalFormatSymbols(Locale.ROOT));

Modified: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/SimpleDVProducer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/SimpleDVProducer.java?rev=1416914&r1=1416913&r2=1416914&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/SimpleDVProducer.java
(original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/SimpleDVProducer.java
Tue Dec  4 12:37:08 2012
@@ -37,4 +37,6 @@ public abstract class SimpleDVProducer i
   public abstract BinaryDocValues getBinary(FieldInfo field) throws IOException;
 
   public abstract SortedDocValues getSorted(FieldInfo field) throws IOException;
+
+  public abstract SimpleDVProducer clone();
 }

Modified: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41DocValuesProducer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41DocValuesProducer.java?rev=1416914&r1=1416913&r2=1416914&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41DocValuesProducer.java
(original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41DocValuesProducer.java
Tue Dec  4 12:37:08 2012
@@ -81,6 +81,12 @@ public class Lucene41DocValuesProducer e
       IOUtils.close(cfs);
     }
   }
+
+  @Override
+  public SimpleDVProducer clone() {
+    // nocommit todo
+    return null;
+  }
   
   @Override
   public NumericDocValues getNumeric(FieldInfo field) throws IOException {

Modified: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldDocValuesFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldDocValuesFormat.java?rev=1416914&r1=1416913&r2=1416914&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldDocValuesFormat.java
(original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldDocValuesFormat.java
Tue Dec  4 12:37:08 2012
@@ -20,6 +20,7 @@ package org.apache.lucene.codecs.perfiel
 import java.io.Closeable;
 import java.io.IOException;
 import java.util.HashMap;
+import java.util.IdentityHashMap;
 import java.util.Map;
 import java.util.ServiceLoader; // javadocs
 import java.util.TreeMap;
@@ -183,6 +184,9 @@ public abstract class PerFieldDocValuesF
     }
   }
 
+  // nocommit what if SimpleNormsFormat wants to use this
+  // ...?  we have a "boolean isNorms" issue...?  I guess we
+  // just need to make a PerFieldNormsFormat?
   private class FieldsReader extends SimpleDVProducer {
 
     private final Map<String,SimpleDVProducer> fields = new TreeMap<String,SimpleDVProducer>();
@@ -219,6 +223,24 @@ public abstract class PerFieldDocValuesF
       }
     }
 
+    private FieldsReader(FieldsReader other) {
+
+      Map<SimpleDVProducer,SimpleDVProducer> oldToNew = new IdentityHashMap<SimpleDVProducer,SimpleDVProducer>();
+      // First clone all formats
+      for(Map.Entry<String,SimpleDVProducer> ent : other.formats.entrySet()) {
+        SimpleDVProducer values = ent.getValue().clone();
+        formats.put(ent.getKey(), values);
+        oldToNew.put(ent.getValue(), values);
+      }
+
+      // Then rebuild fields:
+      for(Map.Entry<String,SimpleDVProducer> ent : other.fields.entrySet()) {
+        SimpleDVProducer producer = oldToNew.get(ent.getValue());
+        assert producer != null;
+        fields.put(ent.getKey(), producer);
+      }
+    }
+
     @Override
     public NumericDocValues getNumeric(FieldInfo field) throws IOException {
       SimpleDVProducer producer = fields.get(field.name);
@@ -241,6 +263,11 @@ public abstract class PerFieldDocValuesF
     public void close() throws IOException {
       IOUtils.close(formats.values());
     }
+
+    @Override
+    public SimpleDVProducer clone() {
+      return new FieldsReader(this);
+    }
   }
 
   @Override

Modified: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/AtomicReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/AtomicReader.java?rev=1416914&r1=1416913&r2=1416914&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/AtomicReader.java
(original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/AtomicReader.java
Tue Dec  4 12:37:08 2012
@@ -163,16 +163,19 @@ public abstract class AtomicReader exten
    */
   public abstract DocValues docValues(String field) throws IOException;
 
+  // nocommit document that these are thread-private:
   /** Returns {@link NumericDocValues} for this field, or
    *  null if no {@link NumericDocValues} were indexed for
    *  this field. */
   public abstract NumericDocValues getNumericDocValues(String field) throws IOException;
 
+  // nocommit document that these are thread-private:
   /** Returns {@link BinaryDocValues} for this field, or
    *  null if no {@link BinaryDocValues} were indexed for
    *  this field. */
   public abstract BinaryDocValues getBinaryDocValues(String field) throws IOException;
 
+  // nocommit document that these are thread-private:
   /** Returns {@link SortedDocValues} for this field, or
    *  null if no {@link SortedDocValues} were indexed for
    *  this field. */
@@ -184,6 +187,7 @@ public abstract class AtomicReader exten
    */
   public abstract DocValues normValues(String field) throws IOException;
 
+  // nocommit document that these are thread-private:
   /** Returns {@link NumericDocValues} representing norms
    *  for this field, or null if no {@link NumericDocValues}
    *  were indexed. */

Modified: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/SegmentCoreReaders.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/SegmentCoreReaders.java?rev=1416914&r1=1416913&r2=1416914&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/SegmentCoreReaders.java
(original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/SegmentCoreReaders.java
Tue Dec  4 12:37:08 2012
@@ -65,6 +65,9 @@ final class SegmentCoreReaders {
   final TermVectorsReader termVectorsReaderOrig;
   final CompoundFileDirectory cfsReader;
 
+  // nocommit we should make a single thread local w/ a
+  // class holding these N things...?
+
   final CloseableThreadLocal<StoredFieldsReader> fieldsReaderLocal = new CloseableThreadLocal<StoredFieldsReader>()
{
     @Override
     protected StoredFieldsReader initialValue() {
@@ -75,10 +78,31 @@ final class SegmentCoreReaders {
   final CloseableThreadLocal<TermVectorsReader> termVectorsLocal = new CloseableThreadLocal<TermVectorsReader>()
{
     @Override
     protected TermVectorsReader initialValue() {
-      return (termVectorsReaderOrig == null) ?
-        null : termVectorsReaderOrig.clone();
+      return (termVectorsReaderOrig == null) ? null : termVectorsReaderOrig.clone();
+    }
+  };
+
+  // nocommit not great to hold onto lots-o-ram in a thread
+  // local...?  do we need a "needsClone"/"isThreadSafe"!?
+  final CloseableThreadLocal<SimpleDVProducer> simpleDocValuesLocal = new CloseableThreadLocal<SimpleDVProducer>()
{
+    @Override
+    protected SimpleDVProducer initialValue() {
+      // nocommit remove null check
+      return (simpleDVProducer == null) ? null : simpleDVProducer.clone();
     }
   };
+
+  // nocommit not great to hold onto lots-o-ram in a thread
+  // local...?  do we need a "needsClone"/"isThreadSafe"!?
+  final CloseableThreadLocal<SimpleDVProducer> simpleNormsLocal = new CloseableThreadLocal<SimpleDVProducer>()
{
+    @Override
+    protected SimpleDVProducer initialValue() {
+      // nocommit remove null check
+      return (simpleNormsProducer == null) ? null : simpleNormsProducer.clone();
+    }
+  };
+
+  // nocommit norms too
   
   private final Set<CoreClosedListener> coreClosedListeners = 
       Collections.synchronizedSet(new LinkedHashSet<CoreClosedListener>());
@@ -181,7 +205,7 @@ final class SegmentCoreReaders {
       return null;
     }
 
-    return simpleDVProducer.getNumeric(fi);
+    return simpleDocValuesLocal.get().getNumeric(fi);
   }
 
   BinaryDocValues getBinaryDocValues(String field) throws IOException {
@@ -204,7 +228,7 @@ final class SegmentCoreReaders {
       return null;
     }
 
-    return simpleDVProducer.getBinary(fi);
+    return simpleDocValuesLocal.get().getBinary(fi);
   }
 
   SortedDocValues getSortedDocValues(String field) throws IOException {
@@ -227,7 +251,7 @@ final class SegmentCoreReaders {
       return null;
     }
 
-    return simpleDVProducer.getSorted(fi);
+    return simpleDocValuesLocal.get().getSorted(fi);
   }
 
   NumericDocValues getSimpleNormValues(String field) throws IOException {
@@ -243,12 +267,12 @@ final class SegmentCoreReaders {
     if (simpleNormsProducer == null) {
       return null;
     }
-    return simpleNormsProducer.getNumeric(fi);
+    return simpleNormsLocal.get().getNumeric(fi);
   }
 
   void decRef() throws IOException {
     if (ref.decrementAndGet() == 0) {
-      IOUtils.close(termVectorsLocal, fieldsReaderLocal, fields, simpleDVProducer,
+      IOUtils.close(termVectorsLocal, fieldsReaderLocal, simpleDocValuesLocal, simpleNormsLocal,
fields, simpleDVProducer,
                     perDocProducer, termVectorsReaderOrig, fieldsReaderOrig, cfsReader, norms,
                     simpleNormsProducer);
       notifyCoreClosedListeners();

Modified: lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/index/TestDocValuesIndexing.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/index/TestDocValuesIndexing.java?rev=1416914&r1=1416913&r2=1416914&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/index/TestDocValuesIndexing.java
(original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/index/TestDocValuesIndexing.java
Tue Dec  4 12:37:08 2012
@@ -986,8 +986,6 @@ public class TestDocValuesIndexing exten
     writer.close();
     
     final AtomicReader sr = getOnlySegmentReader(r);
-    final SortedDocValues dv = sorted(sr, "stringdv");
-    assertNotNull(dv);
 
     final long END_TIME = System.currentTimeMillis() + (TEST_NIGHTLY ? 30 : 1);
 
@@ -1000,11 +998,8 @@ public class TestDocValuesIndexing exten
           @Override
           public void run() {
             Random random = random();            
-            final SortedDocValues stringDV = dv;
             final SortedDocValues stringDVDirect;
             try {
-              
-              assertNotNull(stringDV);
               stringDVDirect = sr.getSortedDocValues("stringdv");
               assertNotNull(stringDVDirect);
             } catch (IOException ioe) {
@@ -1012,11 +1007,7 @@ public class TestDocValuesIndexing exten
             }
             while(System.currentTimeMillis() < END_TIME) {
               final SortedDocValues source;
-              if (random.nextBoolean()) {
-                source = stringDV;
-              } else {
-                source = stringDVDirect;
-              }
+              source = stringDVDirect;
               final BytesRef scratch = new BytesRef();
 
               for(int iter=0;iter<100;iter++) {

Modified: lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/search/TestSameScoresWithThreads.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/search/TestSameScoresWithThreads.java?rev=1416914&r1=1416913&r2=1416914&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/search/TestSameScoresWithThreads.java
(original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/search/TestSameScoresWithThreads.java
Tue Dec  4 12:37:08 2012
@@ -49,24 +49,31 @@ public class TestSameScoresWithThreads e
     final Directory dir = newDirectory();
     final RandomIndexWriter w = new RandomIndexWriter(random(), dir);
     LineFileDocs docs = new LineFileDocs(random());
-    int bytesToIndex = atLeast(100000);
-    int bytesIndexed = 0;
-    while(bytesIndexed < bytesToIndex) {
+    int charsToIndex = atLeast(100000);
+    int charsIndexed = 0;
+    //System.out.println("bytesToIndex=" + charsToIndex);
+    while(charsIndexed < charsToIndex) {
       Document doc = docs.nextDoc();
-      bytesIndexed += RamUsageEstimator.sizeOf(doc);
+      charsIndexed += doc.get("body").length();
       w.addDocument(doc);
+      //System.out.println("  bytes=" + charsIndexed + " add: " + doc);
     }
     IndexReader r = w.getReader();
+    //System.out.println("numDocs=" + r.numDocs());
     w.close();
 
     final IndexSearcher s = new IndexSearcher(r);
     Terms terms = MultiFields.getFields(r).terms("body");
-    long termCount = terms.size();
+    int termCount = 0;
+    TermsEnum termsEnum = terms.iterator(null);
+    while(termsEnum.next() != null) {
+      termCount++;
+    }
     assertTrue(termCount > 0);
     
     // Target ~10 terms to search:
     double chance = 10.0 / termCount;
-    TermsEnum termsEnum = terms.iterator(null);
+    termsEnum = terms.iterator(termsEnum);
     final Map<BytesRef,TopDocs> answers = new HashMap<BytesRef,TopDocs>();
     while(termsEnum.next() != null) {
       if (random().nextDouble() <= chance) {
@@ -90,10 +97,10 @@ public class TestSameScoresWithThreads e
                   List<Map.Entry<BytesRef,TopDocs>> shuffled = new ArrayList<Map.Entry<BytesRef,TopDocs>>(answers.entrySet());
                   Collections.shuffle(shuffled);
                   for(Map.Entry<BytesRef,TopDocs> ent : shuffled) {
-                    TopDocs actual = s.search(new TermQuery(new Term("body", ent.getKey())),
10);
+                    TopDocs actual = s.search(new TermQuery(new Term("body", ent.getKey())),
100);
                     TopDocs expected = ent.getValue();
                     assertEquals(expected.totalHits, actual.totalHits);
-                    assertEquals(expected.scoreDocs.length, actual.scoreDocs.length);
+                    assertEquals("query=" + ent.getKey().utf8ToString(), expected.scoreDocs.length,
actual.scoreDocs.length);
                     for(int hit=0;hit<expected.scoreDocs.length;hit++) {
                       assertEquals(expected.scoreDocs[hit].doc, actual.scoreDocs[hit].doc);
                       // Floats really should be identical:



Mime
View raw message