lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mikemcc...@apache.org
Subject svn commit: r1051066 - in /lucene/dev/branches/branch_3x: ./ lucene/ lucene/contrib/misc/src/java/org/apache/lucene/index/ lucene/src/java/org/apache/lucene/index/ lucene/src/test/org/apache/lucene/index/ solr/
Date Mon, 20 Dec 2010 11:14:37 GMT
Author: mikemccand
Date: Mon Dec 20 11:14:36 2010
New Revision: 1051066

URL: http://svn.apache.org/viewvc?rev=1051066&view=rev
Log:
fix IW infoStream to not include doc store files size when reporting flushed postings size
of new segment vs RAM size

Modified:
    lucene/dev/branches/branch_3x/   (props changed)
    lucene/dev/branches/branch_3x/lucene/   (props changed)
    lucene/dev/branches/branch_3x/lucene/contrib/misc/src/java/org/apache/lucene/index/BalancedSegmentMergePolicy.java
    lucene/dev/branches/branch_3x/lucene/contrib/misc/src/java/org/apache/lucene/index/IndexSplitter.java
    lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/CheckIndex.java
    lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java
    lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/LogMergePolicy.java
    lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/MergePolicy.java
    lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentInfo.java
    lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestSizeBoundedOptimize.java
    lucene/dev/branches/branch_3x/solr/   (props changed)

Modified: lucene/dev/branches/branch_3x/lucene/contrib/misc/src/java/org/apache/lucene/index/BalancedSegmentMergePolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/misc/src/java/org/apache/lucene/index/BalancedSegmentMergePolicy.java?rev=1051066&r1=1051065&r2=1051066&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/misc/src/java/org/apache/lucene/index/BalancedSegmentMergePolicy.java
(original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/misc/src/java/org/apache/lucene/index/BalancedSegmentMergePolicy.java
Mon Dec 20 11:14:36 2010
@@ -56,7 +56,7 @@ public class BalancedSegmentMergePolicy 
   
   @Override
   protected long size(SegmentInfo info) throws IOException {
-    long byteSize = info.sizeInBytes();
+    long byteSize = info.sizeInBytes(true);
     float delRatio = (info.docCount <= 0 ? 0.0f : ((float)info.getDelCount() / (float)info.docCount));
     return (info.docCount <= 0 ?  byteSize : (long)((1.0f - delRatio) * byteSize));
   }

Modified: lucene/dev/branches/branch_3x/lucene/contrib/misc/src/java/org/apache/lucene/index/IndexSplitter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/misc/src/java/org/apache/lucene/index/IndexSplitter.java?rev=1051066&r1=1051065&r2=1051066&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/misc/src/java/org/apache/lucene/index/IndexSplitter.java
(original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/misc/src/java/org/apache/lucene/index/IndexSplitter.java
Mon Dec 20 11:14:36 2010
@@ -99,7 +99,7 @@ public class IndexSplitter {
     DecimalFormat formatter = new DecimalFormat("###,###.###");
     for (int x = 0; x < infos.size(); x++) {
       SegmentInfo info = infos.info(x);
-      String sizeStr = formatter.format(info.sizeInBytes());
+      String sizeStr = formatter.format(info.sizeInBytes(true));
       System.out.println(info.name + " " + sizeStr);
     }
   }

Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/CheckIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/CheckIndex.java?rev=1051066&r1=1051065&r2=1051066&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/CheckIndex.java
(original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/CheckIndex.java
Mon Dec 20 11:14:36 2010
@@ -437,8 +437,8 @@ public class CheckIndex {
         segInfoStat.hasProx = info.getHasProx();
         msg("    numFiles=" + info.files().size());
         segInfoStat.numFiles = info.files().size();
-        msg("    size (MB)=" + nf.format(info.sizeInBytes()/(1024.*1024.)));
-        segInfoStat.sizeMB = info.sizeInBytes()/(1024.*1024.);
+        segInfoStat.sizeMB = info.sizeInBytes(true)/(1024.*1024.);
+        msg("    size (MB)=" + nf.format(segInfoStat.sizeMB));
         Map<String,String> diagnostics = info.getDiagnostics();
         segInfoStat.diagnostics = diagnostics;
         if (diagnostics.size() > 0) {

Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java?rev=1051066&r1=1051065&r2=1051066&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java
(original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java
Mon Dec 20 11:14:36 2010
@@ -727,7 +727,7 @@ final class DocumentsWriter {
           threads.add(threadStates[i].consumer);
         }
 
-        final long startNumBytesUsed = bytesUsed();
+        final double startMBUsed = bytesUsed()/1024./1024.;
         consumer.flush(threads, flushState);
 
         hasVectors |= flushState.hasVectors;
@@ -767,11 +767,13 @@ final class DocumentsWriter {
 
         if (infoStream != null) {
           message("flush: segment=" + newSegment);
-          final long newSegmentSize = newSegment.sizeInBytes();
-          String message = "  ramUsed=" + nf.format(startNumBytesUsed/1024./1024.) + " MB"
+
-            " newFlushedSize=" + nf.format(newSegmentSize/1024/1024) + " MB" +
-            " docs/MB=" + nf.format(numDocsInRAM/(newSegmentSize/1024./1024.)) +
-            " new/old=" + nf.format(100.0*newSegmentSize/startNumBytesUsed) + "%";
+          final double newSegmentSizeNoStore = newSegment.sizeInBytes(false)/1024./1024.;
+          final double newSegmentSize = newSegment.sizeInBytes(true)/1024./1024.;
+          String message = "  ramUsed=" + nf.format(startMBUsed) + " MB" +
+            " newFlushedSize=" + nf.format(newSegmentSize) + " MB" +
+            " (" + nf.format(newSegmentSizeNoStore) + " MB w/o doc stores)" +
+            " docs/MB=" + nf.format(numDocsInRAM/newSegmentSize) +
+            " new/old=" + nf.format(100.0*newSegmentSize/startMBUsed) + "%";
           message(message);
         }
 

Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/LogMergePolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/LogMergePolicy.java?rev=1051066&r1=1051065&r2=1051066&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/LogMergePolicy.java
(original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/LogMergePolicy.java
Mon Dec 20 11:14:36 2010
@@ -206,7 +206,7 @@ public abstract class LogMergePolicy ext
   }
   
   protected long sizeBytes(SegmentInfo info) throws IOException {
-    long byteSize = info.sizeInBytes();
+    long byteSize = info.sizeInBytes(true);
     if (calibrateSizeByDeletes) {
       int delCount = writer.get().numDeletedDocs(info);
       double delRatio = (info.docCount <= 0 ? 0.0f : ((float)delCount / (float)info.docCount));

Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/MergePolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/MergePolicy.java?rev=1051066&r1=1051065&r2=1051066&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/MergePolicy.java
(original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/MergePolicy.java
Mon Dec 20 11:14:36 2010
@@ -169,7 +169,7 @@ public abstract class MergePolicy implem
     public long totalBytesSize() throws IOException {
       long total = 0;
       for (SegmentInfo info : segments) {
-        total += info.sizeInBytes();
+        total += info.sizeInBytes(true);
       }
       return total;
     }

Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentInfo.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentInfo.java?rev=1051066&r1=1051065&r2=1051066&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentInfo.java
(original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentInfo.java
Mon Dec 20 11:14:36 2010
@@ -271,13 +271,16 @@ public final class SegmentInfo {
 
   /** Returns total size in bytes of all of files used by
    *  this segment. */
-  public long sizeInBytes() throws IOException {
+  public long sizeInBytes(boolean includeDocStores) throws IOException {
     if (sizeInBytes == -1) {
       List<String> files = files();
       final int size = files.size();
       sizeInBytes = 0;
       for(int i=0;i<size;i++) {
         final String fileName = files.get(i);
+        if (!includeDocStores && IndexFileNames.isDocStoreFile(fileName)) {
+          continue;
+        }
         // We don't count bytes used by a shared doc store
         // against this segment:
         if (docStoreOffset == -1 || !IndexFileNames.isDocStoreFile(fileName))

Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestSizeBoundedOptimize.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestSizeBoundedOptimize.java?rev=1051066&r1=1051065&r2=1051066&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestSizeBoundedOptimize.java
(original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestSizeBoundedOptimize.java
Mon Dec 20 11:14:36 2010
@@ -59,7 +59,7 @@ public class TestSizeBoundedOptimize ext
     
     SegmentInfos sis = new SegmentInfos();
     sis.read(dir);
-    double min = sis.info(0).sizeInBytes();
+    double min = sis.info(0).sizeInBytes(true);
     
     conf = newWriterConfig();
     LogByteSizeMergePolicy lmp = new LogByteSizeMergePolicy();



Mime
View raw message