lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mikemcc...@apache.org
Subject svn commit: r1603492 [1/2] - in /lucene/dev/branches/lucene5752: ./ lucene/ lucene/codecs/ lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/ lucene/codecs/src/java/org/apache/lucene/codecs/bloom/ lucene/codecs/src/java/org/apache/lucene/codec...
Date Wed, 18 Jun 2014 15:46:00 GMT
Author: mikemccand
Date: Wed Jun 18 15:45:58 2014
New Revision: 1603492

URL: http://svn.apache.org/r1603492
Log:
LUCENE-5752: merge trunk

Added:
    lucene/dev/branches/lucene5752/solr/core/src/test/org/apache/solr/cloud/RollingRestartTest.java
      - copied unchanged from r1603490, lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/RollingRestartTest.java
    lucene/dev/branches/lucene5752/solr/core/src/test/org/apache/solr/schema/TestCloudManagedSchemaConcurrent.java
      - copied unchanged from r1603490, lucene/dev/trunk/solr/core/src/test/org/apache/solr/schema/TestCloudManagedSchemaConcurrent.java
Removed:
    lucene/dev/branches/lucene5752/solr/core/src/test/org/apache/solr/schema/TestCloudManagedSchemaAddField.java
    lucene/dev/branches/lucene5752/solr/core/src/test/org/apache/solr/schema/TestCloudManagedSchemaAddFields.java
    lucene/dev/branches/lucene5752/solr/core/src/test/org/apache/solr/schema/TestCloudManagedSchemaCopyFields.java
Modified:
    lucene/dev/branches/lucene5752/   (props changed)
    lucene/dev/branches/lucene5752/lucene/   (props changed)
    lucene/dev/branches/lucene5752/lucene/CHANGES.txt   (contents, props changed)
    lucene/dev/branches/lucene5752/lucene/codecs/   (props changed)
    lucene/dev/branches/lucene5752/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/FixedGapTermsIndexReader.java
    lucene/dev/branches/lucene5752/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/BloomFilteringPostingsFormat.java
    lucene/dev/branches/lucene5752/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectPostingsFormat.java
    lucene/dev/branches/lucene5752/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTOrdTermsReader.java
    lucene/dev/branches/lucene5752/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermOutputs.java
    lucene/dev/branches/lucene5752/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsReader.java
    lucene/dev/branches/lucene5752/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesReader.java
    lucene/dev/branches/lucene5752/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsReader.java
    lucene/dev/branches/lucene5752/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextStoredFieldsReader.java
    lucene/dev/branches/lucene5752/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsReader.java
    lucene/dev/branches/lucene5752/lucene/codecs/src/test/org/apache/lucene/codecs/blockterms/TestFixedGapPostingsFormat.java
    lucene/dev/branches/lucene5752/lucene/codecs/src/test/org/apache/lucene/codecs/memory/TestDirectPostingsFormat.java
    lucene/dev/branches/lucene5752/lucene/codecs/src/test/org/apache/lucene/codecs/memory/TestMemoryPostingsFormat.java
    lucene/dev/branches/lucene5752/lucene/core/   (props changed)
    lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/codecs/blocktree/FieldReader.java
    lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsIndexReader.java
    lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40DocValuesReader.java
    lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40StoredFieldsReader.java
    lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsReader.java
    lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/codecs/lucene42/Lucene42DocValuesProducer.java
    lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldPostingsFormat.java
    lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/index/SegmentCoreReaders.java
    lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/index/SegmentReader.java
    lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/PagedBytes.java
    lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/Version.java
    lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/fst/ByteSequenceOutputs.java
    lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/fst/BytesStore.java
    lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/fst/FST.java
    lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/fst/Outputs.java
    lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/fst/PairOutputs.java
    lucene/dev/branches/lucene5752/lucene/core/src/test/org/apache/lucene/index/TestOrdinalMap.java
    lucene/dev/branches/lucene5752/lucene/core/src/test/org/apache/lucene/util/TestPagedBytes.java
    lucene/dev/branches/lucene5752/lucene/misc/   (props changed)
    lucene/dev/branches/lucene5752/lucene/misc/src/java/org/apache/lucene/index/sorter/SortingAtomicReader.java
    lucene/dev/branches/lucene5752/lucene/misc/src/test/org/apache/lucene/index/sorter/SorterTestBase.java
    lucene/dev/branches/lucene5752/lucene/spatial/   (props changed)
    lucene/dev/branches/lucene5752/lucene/spatial/src/java/org/apache/lucene/spatial/bbox/BBoxStrategy.java
    lucene/dev/branches/lucene5752/lucene/test-framework/   (props changed)
    lucene/dev/branches/lucene5752/lucene/test-framework/src/java/org/apache/lucene/index/BaseIndexFileFormatTestCase.java
    lucene/dev/branches/lucene5752/lucene/test-framework/src/java/org/apache/lucene/index/BaseNormsFormatTestCase.java
    lucene/dev/branches/lucene5752/lucene/test-framework/src/java/org/apache/lucene/util/RamUsageTester.java
    lucene/dev/branches/lucene5752/solr/   (props changed)
    lucene/dev/branches/lucene5752/solr/CHANGES.txt   (contents, props changed)
    lucene/dev/branches/lucene5752/solr/contrib/   (props changed)
    lucene/dev/branches/lucene5752/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DateFormatTransformer.java
    lucene/dev/branches/lucene5752/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestDateFormatTransformer.java
    lucene/dev/branches/lucene5752/solr/core/   (props changed)
    lucene/dev/branches/lucene5752/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java
    lucene/dev/branches/lucene5752/solr/core/src/java/org/apache/solr/cloud/LeaderElector.java
    lucene/dev/branches/lucene5752/solr/core/src/java/org/apache/solr/cloud/Overseer.java
    lucene/dev/branches/lucene5752/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java
    lucene/dev/branches/lucene5752/solr/core/src/java/org/apache/solr/cloud/ZkController.java
    lucene/dev/branches/lucene5752/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java
    lucene/dev/branches/lucene5752/solr/core/src/java/org/apache/solr/request/SimpleFacets.java
    lucene/dev/branches/lucene5752/solr/core/src/java/org/apache/solr/schema/IndexSchema.java
    lucene/dev/branches/lucene5752/solr/core/src/java/org/apache/solr/schema/ManagedIndexSchema.java
    lucene/dev/branches/lucene5752/solr/core/src/test/org/apache/solr/cloud/OverseerRolesTest.java
    lucene/dev/branches/lucene5752/solr/core/src/test/org/apache/solr/handler/component/DistributedDebugComponentTest.java

Modified: lucene/dev/branches/lucene5752/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/CHANGES.txt?rev=1603492&r1=1603491&r2=1603492&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/lucene5752/lucene/CHANGES.txt Wed Jun 18 15:45:58 2014
@@ -94,6 +94,10 @@ Other
 * LUCENE-5563: Removed sep layout: which has fallen behind on features and doesn't
   perform as well as other options.  (Robert Muir)
 
+======================= Lucene 4.10.0 ======================
+
+(No Changes)
+
 ======================= Lucene 4.9.0 =======================
 
 Changes in Runtime Behavior

Modified: lucene/dev/branches/lucene5752/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/FixedGapTermsIndexReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/FixedGapTermsIndexReader.java?rev=1603492&r1=1603491&r2=1603492&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/FixedGapTermsIndexReader.java (original)
+++ lucene/dev/branches/lucene5752/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/FixedGapTermsIndexReader.java Wed Jun 18 15:45:58 2014
@@ -59,8 +59,7 @@ public class FixedGapTermsIndexReader ex
   private final static int PAGED_BYTES_BITS = 15;
 
   // all fields share this single logical byte[]
-  private final PagedBytes termBytes = new PagedBytes(PAGED_BYTES_BITS);
-  private PagedBytes.Reader termBytesReader;
+  private final PagedBytes.Reader termBytesReader;
 
   final HashMap<FieldInfo,FieldIndexData> fields = new HashMap<>();
   
@@ -71,6 +70,7 @@ public class FixedGapTermsIndexReader ex
   
   public FixedGapTermsIndexReader(Directory dir, FieldInfos fieldInfos, String segment, Comparator<BytesRef> termComp, String segmentSuffix, IOContext context)
     throws IOException {
+    final PagedBytes termBytes = new PagedBytes(PAGED_BYTES_BITS);
 
     this.termComp = termComp;
     
@@ -115,7 +115,7 @@ public class FixedGapTermsIndexReader ex
           throw new CorruptIndexException("invalid packedIndexStart: " + packedIndexStart + " indexStart: " + indexStart + "numIndexTerms: " + numIndexTerms + " (resource=" + in + ")");
         }
         final FieldInfo fieldInfo = fieldInfos.fieldInfo(field);
-        FieldIndexData previous = fields.put(fieldInfo, new FieldIndexData(in, indexStart, termsStart, packedIndexStart, packedOffsetsStart, numIndexTerms));
+        FieldIndexData previous = fields.put(fieldInfo, new FieldIndexData(in, termBytes, indexStart, termsStart, packedIndexStart, packedOffsetsStart, numIndexTerms));
         if (previous != null) {
           throw new CorruptIndexException("duplicate field: " + fieldInfo.name + " (resource=" + in + ")");
         }
@@ -238,7 +238,7 @@ public class FixedGapTermsIndexReader ex
     final long numIndexTerms;
     final long termsStart;
     
-    public FieldIndexData(IndexInput in, long indexStart, long termsStart, long packedIndexStart, long packedOffsetsStart, long numIndexTerms) throws IOException {
+    public FieldIndexData(IndexInput in, PagedBytes termBytes, long indexStart, long termsStart, long packedIndexStart, long packedOffsetsStart, long numIndexTerms) throws IOException {
       
       this.termsStart = termsStart;
       termBytesStart = termBytes.getPointer();
@@ -292,9 +292,7 @@ public class FixedGapTermsIndexReader ex
 
   @Override
   public long ramBytesUsed() {
-    long sizeInBytes = ((termBytes!=null) ? termBytes.ramBytesUsed() : 0) + 
-        ((termBytesReader!=null)? termBytesReader.ramBytesUsed() : 0);
-    
+    long sizeInBytes = ((termBytesReader!=null)? termBytesReader.ramBytesUsed() : 0);
     for(FieldIndexData entry : fields.values()) {
       sizeInBytes += entry.ramBytesUsed();
     }

Modified: lucene/dev/branches/lucene5752/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/BloomFilteringPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/BloomFilteringPostingsFormat.java?rev=1603492&r1=1603491&r2=1603492&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/BloomFilteringPostingsFormat.java (original)
+++ lucene/dev/branches/lucene5752/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/BloomFilteringPostingsFormat.java Wed Jun 18 15:45:58 2014
@@ -151,7 +151,7 @@ public final class BloomFilteringPosting
     return new BloomFilteredFieldsProducer(state);
   }
   
-  public class BloomFilteredFieldsProducer extends FieldsProducer {
+  static class BloomFilteredFieldsProducer extends FieldsProducer {
     private FieldsProducer delegateFieldsProducer;
     HashMap<String,FuzzySet> bloomsByFieldName = new HashMap<>();
     

Modified: lucene/dev/branches/lucene5752/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectPostingsFormat.java?rev=1603492&r1=1603491&r2=1603492&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectPostingsFormat.java (original)
+++ lucene/dev/branches/lucene5752/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectPostingsFormat.java Wed Jun 18 15:45:58 2014
@@ -169,11 +169,16 @@ public final class DirectPostingsFormat 
 
   private final static class DirectField extends Terms implements Accountable {
 
+    private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(DirectField.class);
+
     private static abstract class TermAndSkip implements Accountable {
       public int[] skips;
     }
 
     private static final class LowFreqTerm extends TermAndSkip {
+
+      private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(HighFreqTerm.class);
+
       public final int[] postings;
       public final byte[] payloads;
       public final int docFreq;
@@ -188,13 +193,17 @@ public final class DirectPostingsFormat 
 
       @Override
       public long ramBytesUsed() {
-        return ((postings!=null) ? RamUsageEstimator.sizeOf(postings) : 0) + 
+        return BASE_RAM_BYTES_USED +
+            ((postings!=null) ? RamUsageEstimator.sizeOf(postings) : 0) + 
             ((payloads!=null) ? RamUsageEstimator.sizeOf(payloads) : 0);
       }
     }
 
     // TODO: maybe specialize into prx/no-prx/no-frq cases?
     private static final class HighFreqTerm extends TermAndSkip {
+
+      private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(HighFreqTerm.class);
+
       public final long totalTermFreq;
       public final int[] docIDs;
       public final int[] freqs;
@@ -211,19 +220,22 @@ public final class DirectPostingsFormat 
 
       @Override
       public long ramBytesUsed() {
-         long sizeInBytes = 0;
+         long sizeInBytes = BASE_RAM_BYTES_USED;
          sizeInBytes += (docIDs!=null)? RamUsageEstimator.sizeOf(docIDs) : 0;
          sizeInBytes += (freqs!=null)? RamUsageEstimator.sizeOf(freqs) : 0;
          
          if(positions != null) {
+           sizeInBytes += RamUsageEstimator.shallowSizeOf(positions);
            for(int[] position : positions) {
              sizeInBytes += (position!=null) ? RamUsageEstimator.sizeOf(position) : 0;
            }
          }
          
          if (payloads != null) {
+           sizeInBytes += RamUsageEstimator.shallowSizeOf(payloads);
            for(byte[][] payload : payloads) {
              if(payload != null) {
+               sizeInBytes += RamUsageEstimator.shallowSizeOf(payload);
                for(byte[] pload : payload) {
                  sizeInBytes += (pload!=null) ? RamUsageEstimator.sizeOf(pload) : 0; 
                }
@@ -494,7 +506,7 @@ public final class DirectPostingsFormat 
 
     @Override
     public long ramBytesUsed() {
-      long sizeInBytes = 0;
+      long sizeInBytes = BASE_RAM_BYTES_USED;
       sizeInBytes += ((termBytes!=null) ? RamUsageEstimator.sizeOf(termBytes) : 0);
       sizeInBytes += ((termOffsets!=null) ? RamUsageEstimator.sizeOf(termOffsets) : 0);
       sizeInBytes += ((skips!=null) ? RamUsageEstimator.sizeOf(skips) : 0);
@@ -502,6 +514,7 @@ public final class DirectPostingsFormat 
       sizeInBytes += ((sameCounts!=null) ? RamUsageEstimator.sizeOf(sameCounts) : 0);
       
       if(terms!=null) {
+        sizeInBytes += RamUsageEstimator.shallowSizeOf(terms);
         for(TermAndSkip termAndSkip : terms) {
           sizeInBytes += (termAndSkip!=null) ? termAndSkip.ramBytesUsed() : 0;
         }

Modified: lucene/dev/branches/lucene5752/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTOrdTermsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTOrdTermsReader.java?rev=1603492&r1=1603491&r2=1603492&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTOrdTermsReader.java (original)
+++ lucene/dev/branches/lucene5752/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTOrdTermsReader.java Wed Jun 18 15:45:58 2014
@@ -827,7 +827,7 @@ public class FSTOrdTermsReader extends F
   
   @Override
   public long ramBytesUsed() {
-    long ramBytesUsed = 0;
+    long ramBytesUsed = postingsReader.ramBytesUsed();
     for (TermsReader r : fields.values()) {
       if (r.index != null) {
         ramBytesUsed += r.index.ramBytesUsed();

Modified: lucene/dev/branches/lucene5752/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermOutputs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermOutputs.java?rev=1603492&r1=1603491&r2=1603492&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermOutputs.java (original)
+++ lucene/dev/branches/lucene5752/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermOutputs.java Wed Jun 18 15:45:58 2014
@@ -24,6 +24,8 @@ import org.apache.lucene.index.FieldInfo
 import org.apache.lucene.index.FieldInfo.IndexOptions;
 import org.apache.lucene.store.DataInput;
 import org.apache.lucene.store.DataOutput;
+import org.apache.lucene.util.Accountable;
+import org.apache.lucene.util.RamUsageEstimator;
 import org.apache.lucene.util.fst.Outputs;
 
 /**
@@ -46,7 +48,8 @@ class FSTTermOutputs extends Outputs<FST
    * On an FST, only long[] part is 'shared' and pushed towards root.
    * byte[] and term stats will be kept on deeper arcs.
    */
-  static class TermData {
+  static class TermData implements Accountable {
+    private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(TermData.class);
     long[] longs;
     byte[] bytes;
     int docFreq;
@@ -64,6 +67,18 @@ class FSTTermOutputs extends Outputs<FST
       this.totalTermFreq = totalTermFreq;
     }
 
+    @Override
+    public long ramBytesUsed() {
+      long ramBytesUsed = BASE_RAM_BYTES_USED;
+      if (longs != null) {
+        ramBytesUsed += RamUsageEstimator.sizeOf(longs);
+      }
+      if (bytes != null) {
+        ramBytesUsed += RamUsageEstimator.sizeOf(bytes);
+      }
+      return ramBytesUsed;
+    }
+
     // NOTE: actually, FST nodes are seldom 
     // identical when outputs on their arcs 
     // aren't NO_OUTPUTs.
@@ -113,6 +128,11 @@ class FSTTermOutputs extends Outputs<FST
   }
 
   @Override
+  public long ramBytesUsed(TermData output) {
+    return output.ramBytesUsed();
+  }
+
+  @Override
   //
   // The return value will be the smaller one, when these two are 
   // 'comparable', i.e. 

Modified: lucene/dev/branches/lucene5752/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsReader.java?rev=1603492&r1=1603491&r2=1603492&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsReader.java (original)
+++ lucene/dev/branches/lucene5752/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsReader.java Wed Jun 18 15:45:58 2014
@@ -41,6 +41,7 @@ import org.apache.lucene.store.IndexInpu
 import org.apache.lucene.util.ArrayUtil;
 import org.apache.lucene.util.automaton.ByteRunAutomaton;
 import org.apache.lucene.util.automaton.CompiledAutomaton;
+import org.apache.lucene.util.Accountable;
 import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.IOUtils;
@@ -165,7 +166,9 @@ public class FSTTermsReader extends Fiel
     }
   }
 
-  final class TermsReader extends Terms {
+  private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(TermsReader.class);
+  final class TermsReader extends Terms implements Accountable {
+
     final FieldInfo fieldInfo;
     final long numTerms;
     final long sumTotalTermFreq;
@@ -185,6 +188,11 @@ public class FSTTermsReader extends Fiel
     }
 
     @Override
+    public long ramBytesUsed() {
+      return BASE_RAM_BYTES_USED + dict.ramBytesUsed();
+    }
+
+    @Override
     public boolean hasFreqs() {
       return fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS) >= 0;
     }
@@ -732,9 +740,9 @@ public class FSTTermsReader extends Fiel
 
   @Override
   public long ramBytesUsed() {
-    long ramBytesUsed = 0;
+    long ramBytesUsed = postingsReader.ramBytesUsed();
     for (TermsReader r : fields.values()) {
-      ramBytesUsed += r.dict == null ? 0 : r.dict.ramBytesUsed();
+      ramBytesUsed += r.ramBytesUsed();
     }
     return ramBytesUsed;
   }

Modified: lucene/dev/branches/lucene5752/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesReader.java?rev=1603492&r1=1603491&r2=1603492&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesReader.java (original)
+++ lucene/dev/branches/lucene5752/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesReader.java Wed Jun 18 15:45:58 2014
@@ -55,11 +55,17 @@ import org.apache.lucene.store.ChecksumI
 import org.apache.lucene.store.IndexInput;
 import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.RamUsageEstimator;
 import org.apache.lucene.util.StringHelper;
 
 class SimpleTextDocValuesReader extends DocValuesProducer {
 
+  private static final long BASE_RAM_BYTES_USED =
+        RamUsageEstimator.shallowSizeOfInstance(SimpleTextDocValuesReader.class)
+      + RamUsageEstimator.shallowSizeOfInstance(BytesRef.class);
+
   static class OneField {
+    private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(OneField.class);
     long dataStartFilePointer;
     String pattern;
     String ordPattern;
@@ -500,7 +506,8 @@ class SimpleTextDocValuesReader extends 
 
   @Override
   public long ramBytesUsed() {
-    return 0;
+    return BASE_RAM_BYTES_USED + RamUsageEstimator.sizeOf(scratch.bytes)
+        + fields.size() * (RamUsageEstimator.NUM_BYTES_OBJECT_REF * 2L + OneField.BASE_RAM_BYTES_USED);
   }
 
   @Override

Modified: lucene/dev/branches/lucene5752/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsReader.java?rev=1603492&r1=1603491&r2=1603492&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsReader.java (original)
+++ lucene/dev/branches/lucene5752/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsReader.java Wed Jun 18 15:45:58 2014
@@ -55,6 +55,7 @@ import org.apache.lucene.util.CharsRef;
 import org.apache.lucene.util.FixedBitSet;
 import org.apache.lucene.util.IOUtils;
 import org.apache.lucene.util.IntsRef;
+import org.apache.lucene.util.RamUsageEstimator;
 import org.apache.lucene.util.StringHelper;
 import org.apache.lucene.util.UnicodeUtil;
 import org.apache.lucene.util.fst.Builder;
@@ -65,6 +66,11 @@ import org.apache.lucene.util.fst.Positi
 import org.apache.lucene.util.fst.Util;
 
 class SimpleTextFieldsReader extends FieldsProducer {
+
+  private static final long BASE_RAM_BYTES_USED =
+        RamUsageEstimator.shallowSizeOfInstance(SimpleTextFieldsReader.class)
+      + RamUsageEstimator.shallowSizeOfInstance(TreeMap.class);
+
   private final TreeMap<String,Long> fields;
   private final IndexInput in;
   private final FieldInfos fieldInfos;
@@ -497,6 +503,10 @@ class SimpleTextFieldsReader extends Fie
     }
   }
 
+  private static final long TERMS_BASE_RAM_BYTES_USED =
+        RamUsageEstimator.shallowSizeOfInstance(SimpleTextTerms.class)
+      + RamUsageEstimator.shallowSizeOfInstance(BytesRef.class)
+      + RamUsageEstimator.shallowSizeOfInstance(CharsRef.class);
   private class SimpleTextTerms extends Terms implements Accountable {
     private final long termsStart;
     private final FieldInfo fieldInfo;
@@ -581,7 +591,8 @@ class SimpleTextFieldsReader extends Fie
 
     @Override
     public long ramBytesUsed() {
-      return (fst!=null) ? fst.ramBytesUsed() : 0;
+      return TERMS_BASE_RAM_BYTES_USED + (fst!=null ? fst.ramBytesUsed() : 0)
+          + RamUsageEstimator.sizeOf(scratch.bytes) + RamUsageEstimator.sizeOf(scratchUTF16.chars);
     }
 
     @Override
@@ -643,14 +654,14 @@ class SimpleTextFieldsReader extends Fie
 
   @Override
   synchronized public Terms terms(String field) throws IOException {
-    Terms terms = termsCache.get(field);
+    SimpleTextTerms terms = termsCache.get(field);
     if (terms == null) {
       Long fp = fields.get(field);
       if (fp == null) {
         return null;
       } else {
         terms = new SimpleTextTerms(field, fp, maxDoc);
-        termsCache.put(field, (SimpleTextTerms) terms);
+        termsCache.put(field, terms);
       }
     }
     return terms;
@@ -667,8 +678,8 @@ class SimpleTextFieldsReader extends Fie
   }
 
   @Override
-  public long ramBytesUsed() {
-    long sizeInBytes = 0;
+  public synchronized long ramBytesUsed() {
+    long sizeInBytes = BASE_RAM_BYTES_USED + fields.size() * 2 * RamUsageEstimator.NUM_BYTES_OBJECT_REF;
     for(SimpleTextTerms simpleTextTerms : termsCache.values()) {
       sizeInBytes += (simpleTextTerms!=null) ? simpleTextTerms.ramBytesUsed() : 0;
     }

Modified: lucene/dev/branches/lucene5752/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextStoredFieldsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextStoredFieldsReader.java?rev=1603492&r1=1603491&r2=1603492&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextStoredFieldsReader.java (original)
+++ lucene/dev/branches/lucene5752/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextStoredFieldsReader.java Wed Jun 18 15:45:58 2014
@@ -36,6 +36,7 @@ import org.apache.lucene.util.ArrayUtil;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.CharsRef;
 import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.RamUsageEstimator;
 import org.apache.lucene.util.StringHelper;
 import org.apache.lucene.util.UnicodeUtil;
 
@@ -48,6 +49,12 @@ import static org.apache.lucene.codecs.s
  * @lucene.experimental
  */
 public class SimpleTextStoredFieldsReader extends StoredFieldsReader {
+
+  private static final long BASE_RAM_BYTES_USED =
+        RamUsageEstimator.shallowSizeOfInstance(SimpleTextStoredFieldsReader.class)
+      + RamUsageEstimator.shallowSizeOfInstance(BytesRef.class)
+      + RamUsageEstimator.shallowSizeOfInstance(CharsRef.class);
+
   private long offsets[]; /* docid -> offset in .fld file */
   private IndexInput in;
   private BytesRef scratch = new BytesRef();
@@ -199,7 +206,8 @@ public class SimpleTextStoredFieldsReade
 
   @Override
   public long ramBytesUsed() {
-    return 0;
+    return BASE_RAM_BYTES_USED + RamUsageEstimator.sizeOf(offsets)
+        + RamUsageEstimator.sizeOf(scratch.bytes) + RamUsageEstimator.sizeOf(scratchUTF16.chars);
   }
 
   @Override

Modified: lucene/dev/branches/lucene5752/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsReader.java?rev=1603492&r1=1603491&r2=1603492&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsReader.java (original)
+++ lucene/dev/branches/lucene5752/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsReader.java Wed Jun 18 15:45:58 2014
@@ -43,8 +43,10 @@ import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.CharsRef;
 import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.RamUsageEstimator;
 import org.apache.lucene.util.StringHelper;
 import org.apache.lucene.util.UnicodeUtil;
+
 import static org.apache.lucene.codecs.simpletext.SimpleTextTermVectorsWriter.*;
 
 /**
@@ -54,6 +56,12 @@ import static org.apache.lucene.codecs.s
  * @lucene.experimental
  */
 public class SimpleTextTermVectorsReader extends TermVectorsReader {
+
+  private static final long BASE_RAM_BYTES_USED =
+        RamUsageEstimator.shallowSizeOfInstance(SimpleTextTermVectorsReader.class)
+      + RamUsageEstimator.shallowSizeOfInstance(BytesRef.class)
+      + RamUsageEstimator.shallowSizeOfInstance(CharsRef.class);
+
   private long offsets[]; /* docid -> offset in .vec file */
   private IndexInput in;
   private BytesRef scratch = new BytesRef();
@@ -539,7 +547,7 @@ public class SimpleTextTermVectorsReader
 
   @Override
   public long ramBytesUsed() {
-    return 0;
+    return BASE_RAM_BYTES_USED + RamUsageEstimator.sizeOf(offsets);
   }
 
   @Override

Modified: lucene/dev/branches/lucene5752/lucene/codecs/src/test/org/apache/lucene/codecs/blockterms/TestFixedGapPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/codecs/src/test/org/apache/lucene/codecs/blockterms/TestFixedGapPostingsFormat.java?rev=1603492&r1=1603491&r2=1603492&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/codecs/src/test/org/apache/lucene/codecs/blockterms/TestFixedGapPostingsFormat.java (original)
+++ lucene/dev/branches/lucene5752/lucene/codecs/src/test/org/apache/lucene/codecs/blockterms/TestFixedGapPostingsFormat.java Wed Jun 18 15:45:58 2014
@@ -21,7 +21,6 @@ import org.apache.lucene.codecs.Codec;
 import org.apache.lucene.codecs.lucene41ords.Lucene41WithOrds;
 import org.apache.lucene.index.BasePostingsFormatTestCase;
 import org.apache.lucene.util.TestUtil;
-import org.apache.lucene.util.TestUtil;
 
 /**
  * Basic tests of a PF using FixedGap terms dictionary

Modified: lucene/dev/branches/lucene5752/lucene/codecs/src/test/org/apache/lucene/codecs/memory/TestDirectPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/codecs/src/test/org/apache/lucene/codecs/memory/TestDirectPostingsFormat.java?rev=1603492&r1=1603491&r2=1603492&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/codecs/src/test/org/apache/lucene/codecs/memory/TestDirectPostingsFormat.java (original)
+++ lucene/dev/branches/lucene5752/lucene/codecs/src/test/org/apache/lucene/codecs/memory/TestDirectPostingsFormat.java Wed Jun 18 15:45:58 2014
@@ -20,7 +20,6 @@ package org.apache.lucene.codecs.memory;
 import org.apache.lucene.codecs.Codec;
 import org.apache.lucene.index.BasePostingsFormatTestCase;
 import org.apache.lucene.util.TestUtil;
-import org.apache.lucene.util.TestUtil;
 
 /**
  * Tests DirectPostingsFormat

Modified: lucene/dev/branches/lucene5752/lucene/codecs/src/test/org/apache/lucene/codecs/memory/TestMemoryPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/codecs/src/test/org/apache/lucene/codecs/memory/TestMemoryPostingsFormat.java?rev=1603492&r1=1603491&r2=1603492&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/codecs/src/test/org/apache/lucene/codecs/memory/TestMemoryPostingsFormat.java (original)
+++ lucene/dev/branches/lucene5752/lucene/codecs/src/test/org/apache/lucene/codecs/memory/TestMemoryPostingsFormat.java Wed Jun 18 15:45:58 2014
@@ -20,7 +20,6 @@ package org.apache.lucene.codecs.memory;
 import org.apache.lucene.codecs.Codec;
 import org.apache.lucene.index.BasePostingsFormatTestCase;
 import org.apache.lucene.util.TestUtil;
-import org.apache.lucene.util.TestUtil;
 
 /**
  * Tests MemoryPostingsFormat

Modified: lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/codecs/blocktree/FieldReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/codecs/blocktree/FieldReader.java?rev=1603492&r1=1603491&r2=1603492&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/codecs/blocktree/FieldReader.java (original)
+++ lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/codecs/blocktree/FieldReader.java Wed Jun 18 15:45:58 2014
@@ -27,6 +27,7 @@ import org.apache.lucene.store.ByteArray
 import org.apache.lucene.store.IndexInput;
 import org.apache.lucene.util.Accountable;
 import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.RamUsageEstimator;
 import org.apache.lucene.util.automaton.CompiledAutomaton;
 import org.apache.lucene.util.fst.ByteSequenceOutputs;
 import org.apache.lucene.util.fst.FST;
@@ -34,6 +35,11 @@ import org.apache.lucene.util.fst.FST;
 /** BlockTree's implementation of {@link Terms}. */
 // public for CheckIndex:
 public final class FieldReader extends Terms implements Accountable {
+
+  private static final long BASE_RAM_BYTES_USED =
+      RamUsageEstimator.shallowSizeOfInstance(FieldReader.class)
+      + 3 * RamUsageEstimator.shallowSizeOfInstance(BytesRef.class);
+
   final long numTerms;
   final FieldInfo fieldInfo;
   final long sumTotalTermFreq;
@@ -172,6 +178,6 @@ public final class FieldReader extends T
     
   @Override
   public long ramBytesUsed() {
-    return ((index!=null)? index.ramBytesUsed() : 0);
+    return BASE_RAM_BYTES_USED + ((index!=null)? index.ramBytesUsed() : 0);
   }
 }

Modified: lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsIndexReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsIndexReader.java?rev=1603492&r1=1603491&r2=1603492&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsIndexReader.java (original)
+++ lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsIndexReader.java Wed Jun 18 15:45:58 2014
@@ -36,6 +36,8 @@ import org.apache.lucene.util.packed.Pac
  */
 public final class CompressingStoredFieldsIndexReader implements Cloneable, Accountable {
 
+  private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(CompressingStoredFieldsIndexReader.class);
+
   final int maxDoc;
   final int[] docBases;
   final long[] startPointers;
@@ -163,12 +165,14 @@ public final class CompressingStoredFiel
 
   @Override
   public long ramBytesUsed() {
-    long res = 0;
-    
-    for(PackedInts.Reader r : docBasesDeltas) {
+    long res = BASE_RAM_BYTES_USED;
+
+    res += RamUsageEstimator.shallowSizeOf(docBasesDeltas);
+    for (PackedInts.Reader r : docBasesDeltas) {
       res += r.ramBytesUsed();
     }
-    for(PackedInts.Reader r : startPointersDeltas) {
+    res += RamUsageEstimator.shallowSizeOf(startPointersDeltas);
+    for (PackedInts.Reader r : startPointersDeltas) {
       res += r.ramBytesUsed();
     }
 

Modified: lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40DocValuesReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40DocValuesReader.java?rev=1603492&r1=1603491&r2=1603492&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40DocValuesReader.java (original)
+++ lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40DocValuesReader.java Wed Jun 18 15:45:58 2014
@@ -328,7 +328,7 @@ final class Lucene40DocValuesReader exte
       final PagedBytes.Reader bytesReader = bytes.freeze(true);
       CodecUtil.checkEOF(input);
       success = true;
-      ramBytesUsed.addAndGet(bytes.ramBytesUsed());
+      ramBytesUsed.addAndGet(bytesReader.ramBytesUsed());
       return new BinaryDocValues() {
 
         @Override
@@ -370,7 +370,7 @@ final class Lucene40DocValuesReader exte
       CodecUtil.checkEOF(data);
       CodecUtil.checkEOF(index);
       success = true;
-      ramBytesUsed.addAndGet(bytes.ramBytesUsed() + reader.ramBytesUsed());
+      ramBytesUsed.addAndGet(bytesReader.ramBytesUsed() + reader.ramBytesUsed());
       return new BinaryDocValues() {
         @Override
         public BytesRef get(int docID) {
@@ -414,7 +414,7 @@ final class Lucene40DocValuesReader exte
       final PackedInts.Reader reader = PackedInts.getReader(index);
       CodecUtil.checkEOF(data);
       CodecUtil.checkEOF(index);
-      ramBytesUsed.addAndGet(bytes.ramBytesUsed() + reader.ramBytesUsed());
+      ramBytesUsed.addAndGet(bytesReader.ramBytesUsed() + reader.ramBytesUsed());
       success = true;
       return new BinaryDocValues() {
         @Override
@@ -457,7 +457,7 @@ final class Lucene40DocValuesReader exte
       final PackedInts.Reader reader = PackedInts.getReader(index);
       CodecUtil.checkEOF(data);
       CodecUtil.checkEOF(index);
-      ramBytesUsed.addAndGet(bytes.ramBytesUsed() + reader.ramBytesUsed());
+      ramBytesUsed.addAndGet(bytesReader.ramBytesUsed() + reader.ramBytesUsed());
       success = true;
       return new BinaryDocValues() {
         
@@ -540,7 +540,7 @@ final class Lucene40DocValuesReader exte
     bytes.copy(data, fixedLength * (long) valueCount);
     final PagedBytes.Reader bytesReader = bytes.freeze(true);
     final PackedInts.Reader reader = PackedInts.getReader(index);
-    ramBytesUsed.addAndGet(bytes.ramBytesUsed() + reader.ramBytesUsed());
+    ramBytesUsed.addAndGet(bytesReader.ramBytesUsed() + reader.ramBytesUsed());
 
     return correctBuggyOrds(new SortedDocValues() {
       @Override
@@ -578,7 +578,7 @@ final class Lucene40DocValuesReader exte
     final PackedInts.Reader ordsReader = PackedInts.getReader(index);
 
     final int valueCount = addressReader.size() - 1;
-    ramBytesUsed.addAndGet(bytes.ramBytesUsed() + addressReader.ramBytesUsed() + ordsReader.ramBytesUsed());
+    ramBytesUsed.addAndGet(bytesReader.ramBytesUsed() + addressReader.ramBytesUsed() + ordsReader.ramBytesUsed());
 
     return correctBuggyOrds(new SortedDocValues() {
       @Override

Modified: lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40StoredFieldsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40StoredFieldsReader.java?rev=1603492&r1=1603491&r2=1603492&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40StoredFieldsReader.java (original)
+++ lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40StoredFieldsReader.java Wed Jun 18 15:45:58 2014
@@ -32,6 +32,7 @@ import org.apache.lucene.store.Directory
 import org.apache.lucene.store.IOContext;
 import org.apache.lucene.store.IndexInput;
 import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.RamUsageEstimator;
 
 import java.io.Closeable;
 import java.nio.charset.StandardCharsets;
@@ -47,6 +48,9 @@ import static org.apache.lucene.codecs.l
  * @lucene.internal
  */
 public final class Lucene40StoredFieldsReader extends StoredFieldsReader implements Cloneable, Closeable {
+
+  private static final long RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(Lucene40StoredFieldsReader.class);
+
   private final FieldInfos fieldInfos;
   private final IndexInput fieldsStream;
   private final IndexInput indexStream;
@@ -248,7 +252,7 @@ public final class Lucene40StoredFieldsR
 
   @Override
   public long ramBytesUsed() {
-    return 0;
+    return RAM_BYTES_USED;
   }
 
   @Override

Modified: lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsReader.java?rev=1603492&r1=1603491&r2=1603492&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsReader.java (original)
+++ lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsReader.java Wed Jun 18 15:45:58 2014
@@ -43,6 +43,7 @@ import org.apache.lucene.util.ArrayUtil;
 import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.RamUsageEstimator;
 
 /**
  * Concrete class that reads docId(maybe frq,pos,offset,payloads) list
@@ -53,6 +54,8 @@ import org.apache.lucene.util.IOUtils;
  */
 public final class Lucene41PostingsReader extends PostingsReaderBase {
 
+  private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(Lucene41PostingsReader.class);
+
   private final IndexInput docIn;
   private final IndexInput posIn;
   private final IndexInput payIn;
@@ -1543,7 +1546,7 @@ public final class Lucene41PostingsReade
 
   @Override
   public long ramBytesUsed() {
-    return 0;
+    return BASE_RAM_BYTES_USED;
   }
 
   @Override

Modified: lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/codecs/lucene42/Lucene42DocValuesProducer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/codecs/lucene42/Lucene42DocValuesProducer.java?rev=1603492&r1=1603491&r2=1603492&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/codecs/lucene42/Lucene42DocValuesProducer.java (original)
+++ lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/codecs/lucene42/Lucene42DocValuesProducer.java Wed Jun 18 15:45:58 2014
@@ -291,7 +291,7 @@ class Lucene42DocValuesProducer extends 
     final PagedBytes.Reader bytesReader = bytes.freeze(true);
     if (entry.minLength == entry.maxLength) {
       final int fixedLength = entry.minLength;
-      ramBytesUsed.addAndGet(bytes.ramBytesUsed());
+      ramBytesUsed.addAndGet(bytesReader.ramBytesUsed());
       return new BinaryDocValues() {
         @Override
         public BytesRef get(int docID) {
@@ -302,7 +302,7 @@ class Lucene42DocValuesProducer extends 
       };
     } else {
       final MonotonicBlockPackedReader addresses = MonotonicBlockPackedReader.of(data, entry.packedIntsVersion, entry.blockSize, maxDoc, false);
-      ramBytesUsed.addAndGet(bytes.ramBytesUsed() + addresses.ramBytesUsed());
+      ramBytesUsed.addAndGet(bytesReader.ramBytesUsed() + addresses.ramBytesUsed());
       return new BinaryDocValues() {
 
         @Override

Modified: lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldPostingsFormat.java?rev=1603492&r1=1603491&r2=1603492&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldPostingsFormat.java (original)
+++ lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldPostingsFormat.java Wed Jun 18 15:45:58 2014
@@ -199,7 +199,9 @@ public abstract class PerFieldPostingsFo
     }
   }
 
-  private class FieldsReader extends FieldsProducer {
+  private static class FieldsReader extends FieldsProducer {
+
+    private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(FieldsReader.class);
 
     private final Map<String,FieldsProducer> fields = new TreeMap<>();
     private final Map<String,FieldsProducer> formats = new HashMap<>();
@@ -258,12 +260,13 @@ public abstract class PerFieldPostingsFo
 
     @Override
     public long ramBytesUsed() {
-      long sizeInBytes = 0;
+      long ramBytesUsed = BASE_RAM_BYTES_USED;
+      ramBytesUsed += fields.size() * 2L * RamUsageEstimator.NUM_BYTES_OBJECT_REF;
+      ramBytesUsed += formats.size() * 2L * RamUsageEstimator.NUM_BYTES_OBJECT_REF;
       for(Map.Entry<String,FieldsProducer> entry: formats.entrySet()) {
-        sizeInBytes += entry.getKey().length() * RamUsageEstimator.NUM_BYTES_CHAR;
-        sizeInBytes += entry.getValue().ramBytesUsed();
+        ramBytesUsed += entry.getValue().ramBytesUsed();
       }
-      return sizeInBytes;
+      return ramBytesUsed;
     }
 
     @Override

Modified: lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/index/SegmentCoreReaders.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/index/SegmentCoreReaders.java?rev=1603492&r1=1603491&r2=1603492&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/index/SegmentCoreReaders.java (original)
+++ lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/index/SegmentCoreReaders.java Wed Jun 18 15:45:58 2014
@@ -39,11 +39,14 @@ import org.apache.lucene.store.IOContext
 import org.apache.lucene.util.Accountable;
 import org.apache.lucene.util.CloseableThreadLocal;
 import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.RamUsageEstimator;
 
 /** Holds core readers that are shared (unchanged) when
  * SegmentReader is cloned or reopened */
 final class SegmentCoreReaders implements Accountable {
-  
+
+  private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(SegmentCoreReaders.class);
+
   // Counts how many other readers share the core objects
   // (freqStream, proxStream, tis, etc.) of this reader;
   // when coreRef drops to 0, these core objects may be
@@ -213,7 +216,8 @@ final class SegmentCoreReaders implement
 
   @Override
   public long ramBytesUsed() {
-    return ((normsProducer!=null) ? normsProducer.ramBytesUsed() : 0) +
+    return BASE_RAM_BYTES_USED +
+        ((normsProducer!=null) ? normsProducer.ramBytesUsed() : 0) +
         ((fields!=null) ? fields.ramBytesUsed() : 0) + 
         ((fieldsReaderOrig!=null)? fieldsReaderOrig.ramBytesUsed() : 0) + 
         ((termVectorsReaderOrig!=null) ? termVectorsReaderOrig.ramBytesUsed() : 0);

Modified: lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/index/SegmentReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/index/SegmentReader.java?rev=1603492&r1=1603491&r2=1603492&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/index/SegmentReader.java (original)
+++ lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/index/SegmentReader.java Wed Jun 18 15:45:58 2014
@@ -40,6 +40,7 @@ import org.apache.lucene.util.Accountabl
 import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.CloseableThreadLocal;
 import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.RamUsageEstimator;
 import org.apache.lucene.util.Version;
 
 /**
@@ -51,6 +52,11 @@ import org.apache.lucene.util.Version;
  */
 public final class SegmentReader extends AtomicReader implements Accountable {
 
+  private static final long BASE_RAM_BYTES_USED =
+        RamUsageEstimator.shallowSizeOfInstance(SegmentReader.class)
+      + RamUsageEstimator.shallowSizeOfInstance(SegmentDocValues.class);
+  private static final long LONG_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(Long.class);
+        
   private final SegmentCommitInfo si;
   private final Bits liveDocs;
 
@@ -594,7 +600,10 @@ public final class SegmentReader extends
   @Override
   public long ramBytesUsed() {
     ensureOpen();
-    long ramBytesUsed = 0;
+    long ramBytesUsed = BASE_RAM_BYTES_USED;
+    ramBytesUsed += dvGens.size() * LONG_RAM_BYTES_USED;
+    ramBytesUsed += dvProducers.size() * RamUsageEstimator.NUM_BYTES_OBJECT_REF;
+    ramBytesUsed += dvProducersByField.size() * 2 * RamUsageEstimator.NUM_BYTES_OBJECT_REF;
     if (dvProducers != null) {
       for (DocValuesProducer producer : dvProducers) {
         ramBytesUsed += producer.ramBytesUsed();

Modified: lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/PagedBytes.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/PagedBytes.java?rev=1603492&r1=1603491&r2=1603492&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/PagedBytes.java (original)
+++ lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/PagedBytes.java Wed Jun 18 15:45:58 2014
@@ -18,8 +18,7 @@ package org.apache.lucene.util;
  */
 
 import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
+import java.util.Arrays;
 
 import org.apache.lucene.store.DataInput;
 import org.apache.lucene.store.DataOutput;
@@ -35,9 +34,10 @@ import org.apache.lucene.store.IndexInpu
 // TODO: refactor this, byteblockpool, fst.bytestore, and any
 // other "shift/mask big arrays". there are too many of these classes!
 public final class PagedBytes implements Accountable {
-  private final List<byte[]> blocks = new ArrayList<>();
+  private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(PagedBytes.class);
+  private byte[][] blocks = new byte[16][];
+  private int numBlocks;
   // TODO: these are unused?
-  private final List<Integer> blockEnd = new ArrayList<>();
   private final int blockSize;
   private final int blockBits;
   private final int blockMask;
@@ -54,24 +54,19 @@ public final class PagedBytes implements
    *
    * @see #freeze */
   public final static class Reader implements Accountable {
+    private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(Reader.class);
     private final byte[][] blocks;
-    private final int[] blockEnds;
     private final int blockBits;
     private final int blockMask;
     private final int blockSize;
+    private final long bytesUsedPerBlock;
 
     private Reader(PagedBytes pagedBytes) {
-      blocks = new byte[pagedBytes.blocks.size()][];
-      for(int i=0;i<blocks.length;i++) {
-        blocks[i] = pagedBytes.blocks.get(i);
-      }
-      blockEnds = new int[blocks.length];
-      for(int i=0;i< blockEnds.length;i++) {
-        blockEnds[i] = pagedBytes.blockEnd.get(i);
-      }
+      blocks = Arrays.copyOf(pagedBytes.blocks, pagedBytes.numBlocks);
       blockBits = pagedBytes.blockBits;
       blockMask = pagedBytes.blockMask;
       blockSize = pagedBytes.blockSize;
+      bytesUsedPerBlock = pagedBytes.bytesUsedPerBlock;
     }
 
     /**
@@ -132,7 +127,12 @@ public final class PagedBytes implements
 
     @Override
     public long ramBytesUsed() {
-      return ((blocks!=null) ? (blockSize * blocks.length) : 0);
+      long size = BASE_RAM_BYTES_USED + RamUsageEstimator.shallowSizeOf(blocks);
+      if (blocks.length > 0) {
+        size += (blocks.length - 1) * bytesUsedPerBlock;
+        size += RamUsageEstimator.sizeOf(blocks[blocks.length - 1]);
+      }
+      return size;
     }
   }
 
@@ -144,7 +144,15 @@ public final class PagedBytes implements
     this.blockBits = blockBits;
     blockMask = blockSize-1;
     upto = blockSize;
-    bytesUsedPerBlock = blockSize + RamUsageEstimator.NUM_BYTES_ARRAY_HEADER + RamUsageEstimator.NUM_BYTES_OBJECT_REF;
+    bytesUsedPerBlock = RamUsageEstimator.alignObjectSize(blockSize + RamUsageEstimator.NUM_BYTES_ARRAY_HEADER);
+    numBlocks = 0;
+  }
+
+  private void addBlock(byte[] block) {
+    if (blocks.length == numBlocks) {
+      blocks = Arrays.copyOf(blocks, ArrayUtil.oversize(numBlocks, RamUsageEstimator.NUM_BYTES_OBJECT_REF));
+    }
+    blocks[numBlocks++] = block;
   }
 
   /** Read this many bytes from in */
@@ -153,8 +161,7 @@ public final class PagedBytes implements
       int left = blockSize - upto;
       if (left == 0) {
         if (currentBlock != null) {
-          blocks.add(currentBlock);
-          blockEnd.add(upto);
+          addBlock(currentBlock);
         }
         currentBlock = new byte[blockSize];
         upto = 0;
@@ -179,8 +186,7 @@ public final class PagedBytes implements
     int left = blockSize - upto;
     if (bytes.length > left || currentBlock==null) {
       if (currentBlock != null) {
-        blocks.add(currentBlock);
-        blockEnd.add(upto);
+        addBlock(currentBlock);
         didSkipBytes = true;
       }
       currentBlock = new byte[blockSize];
@@ -214,8 +220,7 @@ public final class PagedBytes implements
     if (currentBlock == null) {
       currentBlock = EMPTY_BYTES;
     }
-    blocks.add(currentBlock);
-    blockEnd.add(upto); 
+    addBlock(currentBlock);
     frozen = true;
     currentBlock = null;
     return new PagedBytes.Reader(this);
@@ -225,13 +230,18 @@ public final class PagedBytes implements
     if (currentBlock == null) {
       return 0;
     } else {
-      return (blocks.size() * ((long) blockSize)) + upto;
+      return (numBlocks * ((long) blockSize)) + upto;
     }
   }
 
   @Override
   public long ramBytesUsed() {
-    return (blocks.size() + (currentBlock != null ? 1 : 0)) * bytesUsedPerBlock;
+    long size = BASE_RAM_BYTES_USED + RamUsageEstimator.shallowSizeOf(blocks)
+        + bytesUsedPerBlock * numBlocks;
+    if (currentBlock != null) {
+      size += RamUsageEstimator.sizeOf(currentBlock);
+    }
+    return size;
   }
 
   /** Copy bytes in, writing the length as a 1 or 2 byte
@@ -247,8 +257,7 @@ public final class PagedBytes implements
         throw new IllegalArgumentException("block size " + blockSize + " is too small to store length " + bytes.length + " bytes");
       }
       if (currentBlock != null) {
-        blocks.add(currentBlock);
-        blockEnd.add(upto);        
+        addBlock(currentBlock);     
       }
       currentBlock = new byte[blockSize];
       upto = 0;
@@ -274,7 +283,7 @@ public final class PagedBytes implements
     private byte[] currentBlock;
 
     PagedBytesDataInput() {
-      currentBlock = blocks.get(0);
+      currentBlock = blocks[0];
     }
 
     @Override
@@ -293,7 +302,7 @@ public final class PagedBytes implements
      *  {@link #getPosition}. */
     public void setPosition(long pos) {
       currentBlockIndex = (int) (pos >> blockBits);
-      currentBlock = blocks.get(currentBlockIndex);
+      currentBlock = blocks[currentBlockIndex];
       currentBlockUpto = (int) (pos & blockMask);
     }
 
@@ -332,7 +341,7 @@ public final class PagedBytes implements
     private void nextBlock() {
       currentBlockIndex++;
       currentBlockUpto = 0;
-      currentBlock = blocks.get(currentBlockIndex);
+      currentBlock = blocks[currentBlockIndex];
     }
   }
 
@@ -341,8 +350,7 @@ public final class PagedBytes implements
     public void writeByte(byte b) {
       if (upto == blockSize) {
         if (currentBlock != null) {
-          blocks.add(currentBlock);
-          blockEnd.add(upto);
+          addBlock(currentBlock);
         }
         currentBlock = new byte[blockSize];
         upto = 0;
@@ -359,8 +367,7 @@ public final class PagedBytes implements
 
       if (upto == blockSize) {
         if (currentBlock != null) {
-          blocks.add(currentBlock);
-          blockEnd.add(upto);
+          addBlock(currentBlock);
         }
         currentBlock = new byte[blockSize];
         upto = 0;
@@ -372,8 +379,7 @@ public final class PagedBytes implements
         final int blockLeft = blockSize - upto;
         if (blockLeft < left) {
           System.arraycopy(b, offset, currentBlock, upto, blockLeft);
-          blocks.add(currentBlock);
-          blockEnd.add(blockSize);
+          addBlock(currentBlock);
           currentBlock = new byte[blockSize];
           upto = 0;
           offset += blockLeft;

Modified: lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/Version.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/Version.java?rev=1603492&r1=1603491&r2=1603492&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/Version.java (original)
+++ lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/Version.java Wed Jun 18 15:45:58 2014
@@ -102,6 +102,13 @@ public enum Version { 
    */
   @Deprecated
   LUCENE_4_9,
+  
+  /**
+   * Match settings and bugs in Lucene's 4.10 release.
+   * @deprecated (5.0) Use latest
+   */
+  @Deprecated
+  LUCENE_4_10,
 
   /** Match settings and bugs in Lucene's 5.0 release.
    *  <p>

Modified: lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/fst/ByteSequenceOutputs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/fst/ByteSequenceOutputs.java?rev=1603492&r1=1603491&r2=1603492&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/fst/ByteSequenceOutputs.java (original)
+++ lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/fst/ByteSequenceOutputs.java Wed Jun 18 15:45:58 2014
@@ -22,6 +22,7 @@ import java.io.IOException;
 import org.apache.lucene.store.DataInput;
 import org.apache.lucene.store.DataOutput;
 import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.RamUsageEstimator;
 
 /**
  * An FST {@link Outputs} implementation where each output
@@ -145,4 +146,9 @@ public final class ByteSequenceOutputs e
   public String outputToString(BytesRef output) {
     return output.toString();
   }
+
+  @Override
+  public long ramBytesUsed(BytesRef output) {
+    return super.ramBytesUsed(output) + RamUsageEstimator.sizeOf(output.bytes);
+  }
 }

Modified: lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/fst/BytesStore.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/fst/BytesStore.java?rev=1603492&r1=1603491&r2=1603492&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/fst/BytesStore.java (original)
+++ lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/fst/BytesStore.java Wed Jun 18 15:45:58 2014
@@ -23,11 +23,17 @@ import java.util.List;
 
 import org.apache.lucene.store.DataInput;
 import org.apache.lucene.store.DataOutput;
+import org.apache.lucene.util.Accountable;
+import org.apache.lucene.util.RamUsageEstimator;
 
 // TODO: merge with PagedBytes, except PagedBytes doesn't
 // let you read while writing which FST needs
 
-class BytesStore extends DataOutput {
+class BytesStore extends DataOutput implements Accountable {
+
+  private static final long BASE_RAM_BYTES_USED =
+        RamUsageEstimator.shallowSizeOfInstance(BytesStore.class)
+      + RamUsageEstimator.shallowSizeOfInstance(ArrayList.class);
 
   private final List<byte[]> blocks = new ArrayList<>();
 
@@ -465,4 +471,14 @@ class BytesStore extends DataOutput {
       }
     };
   }
+
+  @Override
+  public long ramBytesUsed() {
+    long size = BASE_RAM_BYTES_USED;
+    for (byte[] block : blocks) {
+      size += RamUsageEstimator.sizeOf(block);
+    }
+    return size;
+  }
+
 }

Modified: lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/fst/FST.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/fst/FST.java?rev=1603492&r1=1603491&r2=1603492&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/fst/FST.java (original)
+++ lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/fst/FST.java Wed Jun 18 15:45:58 2014
@@ -41,6 +41,7 @@ import org.apache.lucene.util.Constants;
 import org.apache.lucene.util.IOUtils;
 import org.apache.lucene.util.IntsRef;
 import org.apache.lucene.util.PriorityQueue;
+import org.apache.lucene.util.RamUsageEstimator;
 import org.apache.lucene.util.fst.Builder.UnCompiledNode;
 import org.apache.lucene.util.packed.GrowableWriter;
 import org.apache.lucene.util.packed.PackedInts;
@@ -69,6 +70,10 @@ import org.apache.lucene.util.packed.Pac
  * @lucene.experimental
  */
 public final class FST<T> implements Accountable {
+
+  private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(FST.class);
+  private static final long ARC_SHALLOW_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(Arc.class);
+
   /** Specifies allowed range of each int input label for
    *  this FST. */
   public static enum INPUT_TYPE {BYTE1, BYTE2, BYTE4};
@@ -390,15 +395,38 @@ public final class FST<T> implements Acc
     return inputType;
   }
 
+  private long ramBytesUsed(Arc<T>[] arcs) {
+    long size = 0;
+    if (arcs != null) {
+      size += RamUsageEstimator.shallowSizeOf(arcs);
+      for (Arc<T> arc : arcs) {
+        if (arc != null) {
+          size += ARC_SHALLOW_RAM_BYTES_USED;
+          if (arc.output != null && arc.output != outputs.getNoOutput()) {
+            size += outputs.ramBytesUsed(arc.output);
+          }
+          if (arc.nextFinalOutput != null && arc.nextFinalOutput != outputs.getNoOutput()) {
+            size += outputs.ramBytesUsed(arc.nextFinalOutput);
+          }
+        }
+      }
+    }
+    return size;
+  }
+
   @Override
   public long ramBytesUsed() {
-    long size = bytes.getPosition();
+    long size = BASE_RAM_BYTES_USED;
+    size += bytes.ramBytesUsed();
     if (packed) {
       size += nodeRefToAddress.ramBytesUsed();
     } else if (nodeAddress != null) {
       size += nodeAddress.ramBytesUsed();
       size += inCounts.ramBytesUsed();
     }
+    size += ramBytesUsed(cachedRootArcs);
+    size += ramBytesUsed(assertingCachedRootArcs);
+    size += RamUsageEstimator.sizeOf(bytesPerArc);
     return size;
   }
 

Modified: lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/fst/Outputs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/fst/Outputs.java?rev=1603492&r1=1603491&r2=1603492&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/fst/Outputs.java (original)
+++ lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/fst/Outputs.java Wed Jun 18 15:45:58 2014
@@ -21,6 +21,8 @@ import java.io.IOException;
 
 import org.apache.lucene.store.DataInput;
 import org.apache.lucene.store.DataOutput;
+import org.apache.lucene.util.Accountable;
+import org.apache.lucene.util.RamUsageEstimator;
 
 /**
  * Represents the outputs for an FST, providing the basic
@@ -95,4 +97,10 @@ public abstract class Outputs<T> {
   public T merge(T first, T second) {
     throw new UnsupportedOperationException();
   }
+
+  /** Return memory usage for the provided output.
+   *  @see Accountable */
+  public long ramBytesUsed(T output) {
+    return RamUsageEstimator.shallowSizeOf(output);
+  }
 }

Modified: lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/fst/PairOutputs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/fst/PairOutputs.java?rev=1603492&r1=1603491&r2=1603492&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/fst/PairOutputs.java (original)
+++ lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/fst/PairOutputs.java Wed Jun 18 15:45:58 2014
@@ -175,4 +175,16 @@ public class PairOutputs<A,B> extends Ou
   public String toString() {
     return "PairOutputs<" + outputs1 + "," + outputs2 + ">";
   }
+
+  @Override
+  public long ramBytesUsed(Pair<A,B> output) {
+    long ramBytesUsed = super.ramBytesUsed(output);
+    if (output.output1 != null) {
+      ramBytesUsed += outputs1.ramBytesUsed(output.output1);
+    }
+    if (output.output2 != null) {
+      ramBytesUsed += outputs2.ramBytesUsed(output.output2);
+    }
+    return ramBytesUsed;
+  }
 }

Modified: lucene/dev/branches/lucene5752/lucene/core/src/test/org/apache/lucene/index/TestOrdinalMap.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/core/src/test/org/apache/lucene/index/TestOrdinalMap.java?rev=1603492&r1=1603491&r2=1603492&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/core/src/test/org/apache/lucene/index/TestOrdinalMap.java (original)
+++ lucene/dev/branches/lucene5752/lucene/core/src/test/org/apache/lucene/index/TestOrdinalMap.java Wed Jun 18 15:45:58 2014
@@ -19,6 +19,7 @@ package org.apache.lucene.index;
 
 import java.io.IOException;
 import java.lang.reflect.Field;
+import java.util.HashMap;
 
 import org.apache.lucene.analysis.MockAnalyzer;
 import org.apache.lucene.codecs.lucene49.Lucene49DocValuesFormat;
@@ -37,18 +38,28 @@ import org.apache.lucene.util.TestUtil;
 
 public class TestOrdinalMap extends LuceneTestCase {
 
-  private static final RamUsageTester.Filter ORDINAL_MAP_FILTER = new RamUsageTester.Filter() {
-    @Override
-    public boolean accept(Field field) {
-      if (field.getDeclaringClass().equals(OrdinalMap.class) && field.getName().equals("owner")) {
-        return false;
-      }
-      return true;
+  private static final Field ORDINAL_MAP_OWNER_FIELD;
+  static {
+    try {
+      ORDINAL_MAP_OWNER_FIELD = OrdinalMap.class.getDeclaredField("owner");
+    } catch (Exception e) {
+      throw new Error();
     }
+  }
+
+  private static final RamUsageTester.Accumulator ORDINAL_MAP_ACCUMULATOR = new RamUsageTester.Accumulator() {
 
-    public boolean accept(Object o) {
-      return o != LongValues.IDENTITY;
+    public long accumulateObject(Object o, long shallowSize, java.util.Map<Field,Object> fieldValues, java.util.Collection<Object> queue) {
+      if (o == LongValues.IDENTITY) {
+        return 0L;
+      }
+      if (o instanceof OrdinalMap) {
+        fieldValues = new HashMap<>(fieldValues);
+        fieldValues.remove(ORDINAL_MAP_OWNER_FIELD);
+      }
+      return super.accumulateObject(o, shallowSize, fieldValues, queue);
     }
+
   };
 
   public void testRamBytesUsed() throws IOException {
@@ -77,12 +88,12 @@ public class TestOrdinalMap extends Luce
     SortedDocValues sdv = ar.getSortedDocValues("sdv");
     if (sdv instanceof MultiSortedDocValues) {
       OrdinalMap map = ((MultiSortedDocValues) sdv).mapping;
-      assertEquals(RamUsageTester.sizeOf(map, ORDINAL_MAP_FILTER), map.ramBytesUsed());
+      assertEquals(RamUsageTester.sizeOf(map, ORDINAL_MAP_ACCUMULATOR), map.ramBytesUsed());
     }
     SortedSetDocValues ssdv = ar.getSortedSetDocValues("ssdv");
     if (ssdv instanceof MultiSortedSetDocValues) {
       OrdinalMap map = ((MultiSortedSetDocValues) ssdv).mapping;
-      assertEquals(RamUsageTester.sizeOf(map, ORDINAL_MAP_FILTER), map.ramBytesUsed());
+      assertEquals(RamUsageTester.sizeOf(map, ORDINAL_MAP_ACCUMULATOR), map.ramBytesUsed());
     }
     iw.close();
     r.close();

Modified: lucene/dev/branches/lucene5752/lucene/core/src/test/org/apache/lucene/util/TestPagedBytes.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/core/src/test/org/apache/lucene/util/TestPagedBytes.java?rev=1603492&r1=1603491&r2=1603492&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/core/src/test/org/apache/lucene/util/TestPagedBytes.java (original)
+++ lucene/dev/branches/lucene5752/lucene/core/src/test/org/apache/lucene/util/TestPagedBytes.java Wed Jun 18 15:45:58 2014
@@ -27,6 +27,7 @@ import org.apache.lucene.store.IOContext
 import org.apache.lucene.store.IndexInput;
 import org.apache.lucene.store.IndexOutput;
 import org.apache.lucene.store.MockDirectoryWrapper;
+import org.apache.lucene.util.PagedBytes.Reader;
 import org.junit.Ignore;
 
 public class TestPagedBytes extends LuceneTestCase {
@@ -185,4 +186,17 @@ public class TestPagedBytes extends Luce
     dir.close();
   }
 
+  public void testRamBytesUsed() {
+    final int blockBits = TestUtil.nextInt(random(), 4, 22);
+    PagedBytes b = new PagedBytes(blockBits);
+    final int totalBytes = random().nextInt(10000);
+    for (long pointer = 0; pointer < totalBytes; ) {
+      BytesRef bytes = new BytesRef(TestUtil.randomSimpleString(random(), 10));
+      pointer = b.copyUsingLengthPrefix(bytes);
+    }
+    assertEquals(RamUsageTester.sizeOf(b), b.ramBytesUsed());
+    final PagedBytes.Reader reader = b.freeze(random().nextBoolean());
+    assertEquals(RamUsageTester.sizeOf(reader), reader.ramBytesUsed());
+  }
+
 }

Modified: lucene/dev/branches/lucene5752/lucene/misc/src/java/org/apache/lucene/index/sorter/SortingAtomicReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/misc/src/java/org/apache/lucene/index/sorter/SortingAtomicReader.java?rev=1603492&r1=1603491&r2=1603492&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/misc/src/java/org/apache/lucene/index/sorter/SortingAtomicReader.java (original)
+++ lucene/dev/branches/lucene5752/lucene/misc/src/java/org/apache/lucene/index/sorter/SortingAtomicReader.java Wed Jun 18 15:45:58 2014
@@ -30,10 +30,12 @@ import org.apache.lucene.index.Fields;
 import org.apache.lucene.index.FilterAtomicReader;
 import org.apache.lucene.index.NumericDocValues;
 import org.apache.lucene.index.SortedDocValues;
+import org.apache.lucene.index.SortedNumericDocValues;
 import org.apache.lucene.index.SortedSetDocValues;
 import org.apache.lucene.index.StoredFieldVisitor;
 import org.apache.lucene.index.Terms;
 import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.index.sorter.Sorter.DocMap;
 import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.search.Sort;
 import org.apache.lucene.store.IndexInput;
@@ -222,6 +224,32 @@ public class SortingAtomicReader extends
     }
   }
   
+  private static class SortingSortedNumericDocValues extends SortedNumericDocValues {
+    
+    private final SortedNumericDocValues in;
+    private final Sorter.DocMap docMap;
+    
+    SortingSortedNumericDocValues(SortedNumericDocValues in, DocMap docMap) {
+      this.in = in;
+      this.docMap = docMap;
+    }
+    
+    @Override
+    public int count() {
+      return in.count();
+    }
+    
+    @Override
+    public void setDocument(int doc) {
+      in.setDocument(docMap.newToOld(doc));
+    }
+    
+    @Override
+    public long valueAt(int index) {
+      return in.valueAt(index);
+    }
+  }
+  
   private static class SortingBits implements Bits {
 
     private final Bits in;
@@ -786,6 +814,17 @@ public class SortingAtomicReader extends
     if (oldDocValues == null) return null;
     return new SortingNumericDocValues(oldDocValues, docMap);
   }
+  
+  @Override
+  public SortedNumericDocValues getSortedNumericDocValues(String field)
+      throws IOException {
+    final SortedNumericDocValues oldDocValues = in.getSortedNumericDocValues(field);
+    if (oldDocValues == null) {
+      return null;
+    } else {
+      return new SortingSortedNumericDocValues(oldDocValues, docMap);
+    }
+  }
 
   @Override
   public SortedDocValues getSortedDocValues(String field) throws IOException {

Modified: lucene/dev/branches/lucene5752/lucene/misc/src/test/org/apache/lucene/index/sorter/SorterTestBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/misc/src/test/org/apache/lucene/index/sorter/SorterTestBase.java?rev=1603492&r1=1603491&r2=1603492&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/misc/src/test/org/apache/lucene/index/sorter/SorterTestBase.java (original)
+++ lucene/dev/branches/lucene5752/lucene/misc/src/test/org/apache/lucene/index/sorter/SorterTestBase.java Wed Jun 18 15:45:58 2014
@@ -36,6 +36,7 @@ import org.apache.lucene.document.Field.
 import org.apache.lucene.document.FieldType;
 import org.apache.lucene.document.NumericDocValuesField;
 import org.apache.lucene.document.SortedDocValuesField;
+import org.apache.lucene.document.SortedNumericDocValuesField;
 import org.apache.lucene.document.SortedSetDocValuesField;
 import org.apache.lucene.document.StringField;
 import org.apache.lucene.document.TextField;
@@ -52,6 +53,7 @@ import org.apache.lucene.index.NumericDo
 import org.apache.lucene.index.RandomIndexWriter;
 import org.apache.lucene.index.SlowCompositeReaderWrapper;
 import org.apache.lucene.index.SortedDocValues;
+import org.apache.lucene.index.SortedNumericDocValues;
 import org.apache.lucene.index.SortedSetDocValues;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.index.Terms;
@@ -145,6 +147,7 @@ public abstract class SorterTestBase ext
   protected static final String DOC_POSITIONS_FIELD = "positions";
   protected static final String DOC_POSITIONS_TERM = "$all$";
   protected static final String NUMERIC_DV_FIELD = "numeric";
+  protected static final String SORTED_NUMERIC_DV_FIELD = "sorted_numeric";
   protected static final String NORMS_FIELD = "norm";
   protected static final String BINARY_DV_FIELD = "binary";
   protected static final String SORTED_DV_FIELD = "sorted";
@@ -183,6 +186,10 @@ public abstract class SorterTestBase ext
       doc.add(new SortedSetDocValuesField(SORTED_SET_DV_FIELD, new BytesRef(Integer.toString(id))));
       doc.add(new SortedSetDocValuesField(SORTED_SET_DV_FIELD, new BytesRef(Integer.toString(id + 1))));
     }
+    if (defaultCodecSupportsSortedNumeric()) {
+      doc.add(new SortedNumericDocValuesField(SORTED_NUMERIC_DV_FIELD, id));
+      doc.add(new SortedNumericDocValuesField(SORTED_NUMERIC_DV_FIELD, id + 1));
+    }
     doc.add(new Field(TERM_VECTORS_FIELD, Integer.toString(id), TERM_VECTORS_TYPE));
     return doc;
   }
@@ -389,6 +396,20 @@ public abstract class SorterTestBase ext
   }
   
   @Test
+  public void testSortedNumericDocValuesField() throws Exception {
+    assumeTrue("default codec does not support SORTED_NUMERIC", defaultCodecSupportsSortedNumeric());
+    SortedNumericDocValues dv = reader.getSortedNumericDocValues(SORTED_NUMERIC_DV_FIELD);
+    int maxDoc = reader.maxDoc();
+    for (int i = 0; i < maxDoc; i++) {
+      dv.setDocument(i);
+      assertEquals(2, dv.count());
+      int value = sortedValues[i].intValue();
+      assertEquals("incorrect sorted-numeric DocValues for doc " + i, value, dv.valueAt(0));
+      assertEquals("incorrect sorted-numeric DocValues for doc " + i, value + 1, dv.valueAt(1));
+    }
+  }
+  
+  @Test
   public void testTermVectors() throws Exception {
     int maxDoc = reader.maxDoc();
     for (int i = 0; i < maxDoc; i++) {



Mime
View raw message