lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jpou...@apache.org
Subject svn commit: r1570243 - in /lucene/dev/branches/lucene_solr_4_7: ./ lucene/ lucene/core/ lucene/core/src/java/org/apache/lucene/codecs/lucene3x/
Date Thu, 20 Feb 2014 15:50:45 GMT
Author: jpountz
Date: Thu Feb 20 15:50:44 2014
New Revision: 1570243

URL: http://svn.apache.org/r1570243
Log:
LUCENE-5462: Don't use RamUsageEstimator.sizeOf(Object) in codec producers.

Modified:
    lucene/dev/branches/lucene_solr_4_7/   (props changed)
    lucene/dev/branches/lucene_solr_4_7/lucene/   (props changed)
    lucene/dev/branches/lucene_solr_4_7/lucene/CHANGES.txt   (contents, props changed)
    lucene/dev/branches/lucene_solr_4_7/lucene/core/   (props changed)
    lucene/dev/branches/lucene_solr_4_7/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xFields.java
    lucene/dev/branches/lucene_solr_4_7/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xNormsProducer.java
    lucene/dev/branches/lucene_solr_4_7/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xStoredFieldsReader.java
    lucene/dev/branches/lucene_solr_4_7/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xTermVectorsReader.java
    lucene/dev/branches/lucene_solr_4_7/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/TermInfosReader.java
    lucene/dev/branches/lucene_solr_4_7/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/TermInfosReaderIndex.java

Modified: lucene/dev/branches/lucene_solr_4_7/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_7/lucene/CHANGES.txt?rev=1570243&r1=1570242&r2=1570243&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_7/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/lucene_solr_4_7/lucene/CHANGES.txt Thu Feb 20 15:50:44 2014
@@ -187,6 +187,10 @@ Bug fixes
 
 * LUCENE-5447: StandardTokenizer should break at consecutive chars matching
   Word_Break = MidLetter, MidNum and/or MidNumLet (Steve Rowe)
+
+* LUCENE-5462: RamUsageEstimator.sizeOf(Object) is not used anymore to
+  estimate memory usage of segments. This used to make
+  SegmentReader.ramBytesUsed very CPU-intensive. (Adrien Grand)
   
 API Changes
 

Modified: lucene/dev/branches/lucene_solr_4_7/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xFields.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_7/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xFields.java?rev=1570243&r1=1570242&r2=1570243&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_7/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xFields.java
(original)
+++ lucene/dev/branches/lucene_solr_4_7/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xFields.java
Thu Feb 20 15:50:44 2014
@@ -1077,6 +1077,11 @@ class Lucene3xFields extends FieldsProdu
   
   @Override
   public long ramBytesUsed() {
-    return RamUsageEstimator.sizeOf(this);
+    if (tis != null) {
+      return tis.ramBytesUsed();
+    } else {
+      // when there is no index, there is almost nothing loaded into RAM
+      return 0L;
+    }
   }
 }

Modified: lucene/dev/branches/lucene_solr_4_7/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xNormsProducer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_7/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xNormsProducer.java?rev=1570243&r1=1570242&r2=1570243&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_7/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xNormsProducer.java
(original)
+++ lucene/dev/branches/lucene_solr_4_7/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xNormsProducer.java
Thu Feb 20 15:50:44 2014
@@ -23,6 +23,7 @@ import java.util.HashMap;
 import java.util.IdentityHashMap;
 import java.util.Map;
 import java.util.Set;
+import java.util.concurrent.atomic.AtomicLong;
 
 import org.apache.lucene.codecs.DocValuesProducer;
 import org.apache.lucene.index.BinaryDocValues;
@@ -66,6 +67,8 @@ class Lucene3xNormsProducer extends DocV
   IndexInput singleNormStream;
   final int maxdoc;
   
+  private final AtomicLong ramBytesUsed;
+
   // note: just like segmentreader in 3.x, we open up all the files here (including separate
norms) up front.
   // but we just don't do any seeks or reading yet.
   public Lucene3xNormsProducer(Directory dir, SegmentInfo info, FieldInfos fields, IOContext
context) throws IOException {
@@ -125,6 +128,7 @@ class Lucene3xNormsProducer extends DocV
         IOUtils.closeWhileHandlingException(openFiles);
       }
     }
+    ramBytesUsed = new AtomicLong();
   }
   
   @Override
@@ -182,6 +186,7 @@ class Lucene3xNormsProducer extends DocV
           openFiles.remove(file);
           file.close();
         }
+        ramBytesUsed.addAndGet(RamUsageEstimator.sizeOf(bytes));
         instance = new NumericDocValues() {
           @Override
           public long get(int docID) {
@@ -222,6 +227,6 @@ class Lucene3xNormsProducer extends DocV
   
   @Override
   public long ramBytesUsed() {
-    return RamUsageEstimator.sizeOf(this);
+    return ramBytesUsed.get();
   }
 }

Modified: lucene/dev/branches/lucene_solr_4_7/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xStoredFieldsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_7/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xStoredFieldsReader.java?rev=1570243&r1=1570242&r2=1570243&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_7/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xStoredFieldsReader.java
(original)
+++ lucene/dev/branches/lucene_solr_4_7/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xStoredFieldsReader.java
Thu Feb 20 15:50:44 2014
@@ -301,6 +301,7 @@ final class Lucene3xStoredFieldsReader e
   
   @Override
   public long ramBytesUsed() {
-    return RamUsageEstimator.sizeOf(this);
+    // everything is stored on disk
+    return 0;
   }
 }

Modified: lucene/dev/branches/lucene_solr_4_7/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xTermVectorsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_7/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xTermVectorsReader.java?rev=1570243&r1=1570242&r2=1570243&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_7/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xTermVectorsReader.java
(original)
+++ lucene/dev/branches/lucene_solr_4_7/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xTermVectorsReader.java
Thu Feb 20 15:50:44 2014
@@ -740,7 +740,8 @@ class Lucene3xTermVectorsReader extends 
 
   @Override
   public long ramBytesUsed() {
-    return RamUsageEstimator.sizeOf(this);
+    // everything is disk-based
+    return 0;
   }
 }
 

Modified: lucene/dev/branches/lucene_solr_4_7/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/TermInfosReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_7/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/TermInfosReader.java?rev=1570243&r1=1570242&r2=1570243&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_7/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/TermInfosReader.java
(original)
+++ lucene/dev/branches/lucene_solr_4_7/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/TermInfosReader.java
Thu Feb 20 15:50:44 2014
@@ -352,4 +352,8 @@ final class TermInfosReader implements C
     get(term, true);
     return getThreadResources().termEnum.clone();
   }
+
+  long ramBytesUsed() {
+    return index == null ? 0 : index.ramBytesUsed();
+  }
 }

Modified: lucene/dev/branches/lucene_solr_4_7/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/TermInfosReaderIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_7/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/TermInfosReaderIndex.java?rev=1570243&r1=1570242&r2=1570243&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_7/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/TermInfosReaderIndex.java
(original)
+++ lucene/dev/branches/lucene_solr_4_7/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/TermInfosReaderIndex.java
Thu Feb 20 15:50:44 2014
@@ -25,6 +25,7 @@ import java.util.List;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.MathUtil;
+import org.apache.lucene.util.RamUsageEstimator;
 import org.apache.lucene.util.PagedBytes.PagedBytesDataInput;
 import org.apache.lucene.util.PagedBytes.PagedBytesDataOutput;
 import org.apache.lucene.util.PagedBytes;
@@ -49,6 +50,7 @@ class TermInfosReaderIndex {
   private final PackedInts.Reader indexToDataOffset;
   private final int indexSize;
   private final int skipInterval;
+  private final long ramBytesUsed;
 
   /**
    * Loads the segment information at segment load time.
@@ -111,6 +113,9 @@ class TermInfosReaderIndex {
     dataPagedBytes.freeze(true);
     dataInput = dataPagedBytes.getDataInput();
     indexToDataOffset = indexToTerms.getMutable();
+
+    ramBytesUsed = fields.length * (RamUsageEstimator.NUM_BYTES_OBJECT_REF + RamUsageEstimator.shallowSizeOfInstance(Term.class))
+        + dataPagedBytes.ramBytesUsed() + indexToDataOffset.ramBytesUsed();
   }
 
   private static int estimatePageBits(long estSize) {
@@ -253,4 +258,9 @@ class TermInfosReaderIndex {
     input.setPosition(indexToDataOffset.get(termIndex));
     return term.field().compareTo(fields[input.readVInt()].field());
   }
+
+  long ramBytesUsed() {
+    return ramBytesUsed;
+  }
+
 }



Mime
View raw message