lucene-java-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mikemcc...@apache.org
Subject svn commit: r901662 - in /lucene/java/trunk: contrib/analyzers/common/src/java/org/tartarus/snowball/ contrib/snowball/ contrib/snowball/src/java/org/tartarus/snowball/ src/java/org/apache/lucene/analysis/ src/java/org/apache/lucene/analysis/tokenattri...
Date Thu, 21 Jan 2010 11:54:51 GMT
Author: mikemccand
Date: Thu Jan 21 11:54:50 2010
New Revision: 901662

URL: http://svn.apache.org/viewvc?rev=901662&view=rev
Log:
LUCENE-2213: rename ArrayUtil.getNextSize -> oversize; tweak how it picks the next size

Added:
    lucene/java/trunk/contrib/snowball/
      - copied from r900774, lucene/java/trunk/contrib/snowball/
    lucene/java/trunk/src/test/org/apache/lucene/util/TestArrayUtil.java   (with props)
Modified:
    lucene/java/trunk/contrib/analyzers/common/src/java/org/tartarus/snowball/SnowballProgram.java
    lucene/java/trunk/contrib/snowball/src/java/org/tartarus/snowball/SnowballProgram.java
    lucene/java/trunk/src/java/org/apache/lucene/analysis/ASCIIFoldingFilter.java
    lucene/java/trunk/src/java/org/apache/lucene/analysis/Token.java
    lucene/java/trunk/src/java/org/apache/lucene/analysis/tokenattributes/TermAttributeImpl.java
    lucene/java/trunk/src/java/org/apache/lucene/index/DocFieldConsumers.java
    lucene/java/trunk/src/java/org/apache/lucene/index/DocFieldProcessorPerThread.java
    lucene/java/trunk/src/java/org/apache/lucene/index/DocumentsWriter.java
    lucene/java/trunk/src/java/org/apache/lucene/index/FormatPostingsTermsConsumer.java
    lucene/java/trunk/src/java/org/apache/lucene/index/NormsWriterPerField.java
    lucene/java/trunk/src/java/org/apache/lucene/index/StoredFieldsWriter.java
    lucene/java/trunk/src/java/org/apache/lucene/index/TermVectorsTermsWriter.java
    lucene/java/trunk/src/java/org/apache/lucene/index/TermsHash.java
    lucene/java/trunk/src/java/org/apache/lucene/util/ArrayUtil.java
    lucene/java/trunk/src/java/org/apache/lucene/util/RamUsageEstimator.java
    lucene/java/trunk/src/test/org/apache/lucene/analysis/TestToken.java
    lucene/java/trunk/src/test/org/apache/lucene/analysis/tokenattributes/TestTermAttributeImpl.java
    lucene/java/trunk/src/test/org/apache/lucene/util/TestIndexableBinaryStringTools.java

Modified: lucene/java/trunk/contrib/analyzers/common/src/java/org/tartarus/snowball/SnowballProgram.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/analyzers/common/src/java/org/tartarus/snowball/SnowballProgram.java?rev=901662&r1=901661&r2=901662&view=diff
==============================================================================
--- lucene/java/trunk/contrib/analyzers/common/src/java/org/tartarus/snowball/SnowballProgram.java
(original)
+++ lucene/java/trunk/contrib/analyzers/common/src/java/org/tartarus/snowball/SnowballProgram.java
Thu Jan 21 11:54:50 2010
@@ -35,6 +35,7 @@
 import java.lang.reflect.InvocationTargetException;
 
 import org.apache.lucene.util.ArrayUtil;
+import org.apache.lucene.util.RamUsageEstimator;
 
 /**
  * This is the rev 502 of the Snowball SVN trunk,
@@ -432,7 +433,7 @@
 	final int newLength = limit + adjustment;
 	//resize if necessary
 	if (newLength > current.length) {
-	  char newBuffer[] = new char[ArrayUtil.getNextSize(newLength)];
+	  char newBuffer[] = new char[ArrayUtil.oversize(newLength, RamUsageEstimator.NUM_BYTES_CHAR)];
 	  System.arraycopy(current, 0, newBuffer, 0, limit);
 	  current = newBuffer;
 	}

Modified: lucene/java/trunk/contrib/snowball/src/java/org/tartarus/snowball/SnowballProgram.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/snowball/src/java/org/tartarus/snowball/SnowballProgram.java?rev=901662&r1=900774&r2=901662&view=diff
==============================================================================
--- lucene/java/trunk/contrib/snowball/src/java/org/tartarus/snowball/SnowballProgram.java
(original)
+++ lucene/java/trunk/contrib/snowball/src/java/org/tartarus/snowball/SnowballProgram.java
Thu Jan 21 11:54:50 2010
@@ -35,6 +35,7 @@
 import java.lang.reflect.InvocationTargetException;
 
 import org.apache.lucene.util.ArrayUtil;
+import org.apache.lucene.util.RamUsageEstimator;
 
 /**
  * This is the rev 502 of the Snowball SVN trunk,
@@ -432,7 +433,7 @@
 	final int newLength = limit + adjustment;
 	//resize if necessary
 	if (newLength > current.length) {
-	  char newBuffer[] = new char[ArrayUtil.getNextSize(newLength)];
+	  char newBuffer[] = new char[ArrayUtil.oversize(newLength, RamUsageEstimator.NUM_BYTES_CHAR)];
 	  System.arraycopy(current, 0, newBuffer, 0, limit);
 	  current = newBuffer;
 	}

Modified: lucene/java/trunk/src/java/org/apache/lucene/analysis/ASCIIFoldingFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/analysis/ASCIIFoldingFilter.java?rev=901662&r1=901661&r2=901662&view=diff
==============================================================================
--- lucene/java/trunk/src/java/org/apache/lucene/analysis/ASCIIFoldingFilter.java (original)
+++ lucene/java/trunk/src/java/org/apache/lucene/analysis/ASCIIFoldingFilter.java Thu Jan
21 11:54:50 2010
@@ -1,10 +1,5 @@
 package org.apache.lucene.analysis;
 
-import java.io.IOException;
-
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
-import org.apache.lucene.util.ArrayUtil;
-
 /**
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
@@ -22,6 +17,12 @@
  * limitations under the License.
  */
 
+import java.io.IOException;
+
+import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.util.ArrayUtil;
+import org.apache.lucene.util.RamUsageEstimator;
+
 /**
  * This class converts alphabetic, numeric, and symbolic Unicode characters
  * which are not in the first 127 ASCII characters (the "Basic Latin" Unicode
@@ -101,7 +102,7 @@
     // Worst-case length required:
     final int maxSizeNeeded = 4 * length;
     if (output.length < maxSizeNeeded) {
-      output = new char[ArrayUtil.getNextSize(maxSizeNeeded)];
+      output = new char[ArrayUtil.oversize(maxSizeNeeded, RamUsageEstimator.NUM_BYTES_CHAR)];
     }
 
     outputPos = 0;

Modified: lucene/java/trunk/src/java/org/apache/lucene/analysis/Token.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/analysis/Token.java?rev=901662&r1=901661&r2=901662&view=diff
==============================================================================
--- lucene/java/trunk/src/java/org/apache/lucene/analysis/Token.java (original)
+++ lucene/java/trunk/src/java/org/apache/lucene/analysis/Token.java Thu Jan 21 11:54:50 2010
@@ -29,6 +29,7 @@
 import org.apache.lucene.util.Attribute;
 import org.apache.lucene.util.AttributeSource;
 import org.apache.lucene.util.AttributeImpl;
+import org.apache.lucene.util.RamUsageEstimator;
 
 /** 
   A Token is an occurrence of a term from the text of a field.  It consists of
@@ -347,12 +348,12 @@
   public char[] resizeTermBuffer(int newSize) {
     if (termBuffer == null) {
       // The buffer is always at least MIN_BUFFER_SIZE
-      termBuffer = new char[ArrayUtil.getNextSize(newSize < MIN_BUFFER_SIZE ? MIN_BUFFER_SIZE
: newSize)]; 
+      termBuffer = new char[ArrayUtil.oversize(newSize < MIN_BUFFER_SIZE ? MIN_BUFFER_SIZE
: newSize, RamUsageEstimator.NUM_BYTES_CHAR)]; 
     } else {
       if(termBuffer.length < newSize){
         // Not big enough; create a new array with slight
         // over allocation and preserve content
-        final char[] newCharBuffer = new char[ArrayUtil.getNextSize(newSize)];
+        final char[] newCharBuffer = new char[ArrayUtil.oversize(newSize, RamUsageEstimator.NUM_BYTES_CHAR)];
         System.arraycopy(termBuffer, 0, newCharBuffer, 0, termBuffer.length);
         termBuffer = newCharBuffer;
       }
@@ -367,19 +368,19 @@
   private void growTermBuffer(int newSize) {
     if (termBuffer == null) {
       // The buffer is always at least MIN_BUFFER_SIZE    
-      termBuffer = new char[ArrayUtil.getNextSize(newSize < MIN_BUFFER_SIZE ? MIN_BUFFER_SIZE
: newSize)];   
+      termBuffer = new char[ArrayUtil.oversize(newSize < MIN_BUFFER_SIZE ? MIN_BUFFER_SIZE
: newSize, RamUsageEstimator.NUM_BYTES_CHAR)];
     } else {
       if(termBuffer.length < newSize){
         // Not big enough; create a new array with slight
         // over allocation:
-        termBuffer = new char[ArrayUtil.getNextSize(newSize)];
+        termBuffer = new char[ArrayUtil.oversize(newSize, RamUsageEstimator.NUM_BYTES_CHAR)];
       }
     } 
   }
   
   private void initTermBuffer() {
     if (termBuffer == null) {
-      termBuffer = new char[ArrayUtil.getNextSize(MIN_BUFFER_SIZE)];
+      termBuffer = new char[ArrayUtil.oversize(MIN_BUFFER_SIZE, RamUsageEstimator.NUM_BYTES_CHAR)];
       termLength = 0;
     }
   }

Modified: lucene/java/trunk/src/java/org/apache/lucene/analysis/tokenattributes/TermAttributeImpl.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/analysis/tokenattributes/TermAttributeImpl.java?rev=901662&r1=901661&r2=901662&view=diff
==============================================================================
--- lucene/java/trunk/src/java/org/apache/lucene/analysis/tokenattributes/TermAttributeImpl.java
(original)
+++ lucene/java/trunk/src/java/org/apache/lucene/analysis/tokenattributes/TermAttributeImpl.java
Thu Jan 21 11:54:50 2010
@@ -21,6 +21,7 @@
 
 import org.apache.lucene.util.ArrayUtil;
 import org.apache.lucene.util.AttributeImpl;
+import org.apache.lucene.util.RamUsageEstimator;
 
 /**
  * The term text of a Token.
@@ -106,12 +107,12 @@
   public char[] resizeTermBuffer(int newSize) {
     if (termBuffer == null) {
       // The buffer is always at least MIN_BUFFER_SIZE
-      termBuffer = new char[ArrayUtil.getNextSize(newSize < MIN_BUFFER_SIZE ? MIN_BUFFER_SIZE
: newSize)]; 
+      termBuffer = new char[ArrayUtil.oversize(newSize < MIN_BUFFER_SIZE ? MIN_BUFFER_SIZE
: newSize, RamUsageEstimator.NUM_BYTES_CHAR)]; 
     } else {
       if(termBuffer.length < newSize){
         // Not big enough; create a new array with slight
         // over allocation and preserve content
-        final char[] newCharBuffer = new char[ArrayUtil.getNextSize(newSize)];
+        final char[] newCharBuffer = new char[ArrayUtil.oversize(newSize, RamUsageEstimator.NUM_BYTES_CHAR)];
         System.arraycopy(termBuffer, 0, newCharBuffer, 0, termBuffer.length);
         termBuffer = newCharBuffer;
       }
@@ -127,19 +128,19 @@
   private void growTermBuffer(int newSize) {
     if (termBuffer == null) {
       // The buffer is always at least MIN_BUFFER_SIZE
-      termBuffer = new char[ArrayUtil.getNextSize(newSize < MIN_BUFFER_SIZE ? MIN_BUFFER_SIZE
: newSize)];   
+      termBuffer = new char[ArrayUtil.oversize(newSize < MIN_BUFFER_SIZE ? MIN_BUFFER_SIZE
: newSize, RamUsageEstimator.NUM_BYTES_CHAR)];   
     } else {
       if(termBuffer.length < newSize){
         // Not big enough; create a new array with slight
         // over allocation:
-        termBuffer = new char[ArrayUtil.getNextSize(newSize)];
+        termBuffer = new char[ArrayUtil.oversize(newSize, RamUsageEstimator.NUM_BYTES_CHAR)];
       }
     } 
   }
   
   private void initTermBuffer() {
     if (termBuffer == null) {
-      termBuffer = new char[ArrayUtil.getNextSize(MIN_BUFFER_SIZE)];
+      termBuffer = new char[ArrayUtil.oversize(MIN_BUFFER_SIZE, RamUsageEstimator.NUM_BYTES_CHAR)];
       termLength = 0;
     }
   }

Modified: lucene/java/trunk/src/java/org/apache/lucene/index/DocFieldConsumers.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/index/DocFieldConsumers.java?rev=901662&r1=901661&r2=901662&view=diff
==============================================================================
--- lucene/java/trunk/src/java/org/apache/lucene/index/DocFieldConsumers.java (original)
+++ lucene/java/trunk/src/java/org/apache/lucene/index/DocFieldConsumers.java Thu Jan 21 11:54:50
2010
@@ -25,6 +25,7 @@
 import java.io.IOException;
 
 import org.apache.lucene.util.ArrayUtil;
+import org.apache.lucene.util.RamUsageEstimator;
 
 /** This is just a "splitter" class: it lets you wrap two
  *  DocFieldConsumer instances as a single consumer. */
@@ -117,7 +118,7 @@
         // enough space to recycle all outstanding PerDoc
         // instances
         assert allocCount == 1+docFreeList.length;
-        docFreeList = new PerDoc[ArrayUtil.getNextSize(allocCount)];
+        docFreeList = new PerDoc[ArrayUtil.oversize(allocCount, RamUsageEstimator.NUM_BYTES_OBJECT_REF)];
       }
       return new PerDoc();
     } else

Modified: lucene/java/trunk/src/java/org/apache/lucene/index/DocFieldProcessorPerThread.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/index/DocFieldProcessorPerThread.java?rev=901662&r1=901661&r2=901662&view=diff
==============================================================================
--- lucene/java/trunk/src/java/org/apache/lucene/index/DocFieldProcessorPerThread.java (original)
+++ lucene/java/trunk/src/java/org/apache/lucene/index/DocFieldProcessorPerThread.java Thu
Jan 21 11:54:50 2010
@@ -24,6 +24,7 @@
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Fieldable;
 import org.apache.lucene.util.ArrayUtil;
+import org.apache.lucene.util.RamUsageEstimator;
 
 /**
  * Gathers all Fieldables for a document under the same
@@ -340,7 +341,7 @@
         // enough space to recycle all outstanding PerDoc
         // instances
         assert allocCount == 1+docFreeList.length;
-        docFreeList = new PerDoc[ArrayUtil.getNextSize(allocCount)];
+        docFreeList = new PerDoc[ArrayUtil.oversize(allocCount, RamUsageEstimator.NUM_BYTES_OBJECT_REF)];
       }
       return new PerDoc();
     } else

Modified: lucene/java/trunk/src/java/org/apache/lucene/index/DocumentsWriter.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/index/DocumentsWriter.java?rev=901662&r1=901661&r2=901662&view=diff
==============================================================================
--- lucene/java/trunk/src/java/org/apache/lucene/index/DocumentsWriter.java (original)
+++ lucene/java/trunk/src/java/org/apache/lucene/index/DocumentsWriter.java Thu Jan 21 11:54:50
2010
@@ -40,6 +40,7 @@
 import org.apache.lucene.util.ArrayUtil;
 import org.apache.lucene.util.Constants;
 import org.apache.lucene.util.ThreadInterruptedException;
+import org.apache.lucene.util.RamUsageEstimator;
 
 /**
  * This class accepts multiple added documents and directly
@@ -1503,7 +1504,7 @@
         int gap = doc.docID - nextWriteDocID;
         if (gap >= waiting.length) {
           // Grow queue
-          DocWriter[] newArray = new DocWriter[ArrayUtil.getNextSize(gap)];
+          DocWriter[] newArray = new DocWriter[ArrayUtil.oversize(gap, RamUsageEstimator.NUM_BYTES_OBJECT_REF)];
           assert nextWriteLoc >= 0;
           System.arraycopy(waiting, nextWriteLoc, newArray, 0, waiting.length-nextWriteLoc);
           System.arraycopy(waiting, 0, newArray, waiting.length-nextWriteLoc, nextWriteLoc);

Modified: lucene/java/trunk/src/java/org/apache/lucene/index/FormatPostingsTermsConsumer.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/index/FormatPostingsTermsConsumer.java?rev=901662&r1=901661&r2=901662&view=diff
==============================================================================
--- lucene/java/trunk/src/java/org/apache/lucene/index/FormatPostingsTermsConsumer.java (original)
+++ lucene/java/trunk/src/java/org/apache/lucene/index/FormatPostingsTermsConsumer.java Thu
Jan 21 11:54:50 2010
@@ -20,6 +20,7 @@
 import java.io.IOException;
 
 import org.apache.lucene.util.ArrayUtil;
+import org.apache.lucene.util.RamUsageEstimator;
 
 /**
  * NOTE: this API is experimental and will likely change
@@ -35,7 +36,7 @@
   FormatPostingsDocsConsumer addTerm(String text) throws IOException {
     final int len = text.length();
     if (termBuffer == null || termBuffer.length < 1+len)
-      termBuffer = new char[ArrayUtil.getNextSize(1+len)];
+      termBuffer = new char[ArrayUtil.oversize(1+len, RamUsageEstimator.NUM_BYTES_CHAR)];
     text.getChars(0, len, termBuffer, 0);
     termBuffer[len] = 0xffff;
     return addTerm(termBuffer, 0);

Modified: lucene/java/trunk/src/java/org/apache/lucene/index/NormsWriterPerField.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/index/NormsWriterPerField.java?rev=901662&r1=901661&r2=901662&view=diff
==============================================================================
--- lucene/java/trunk/src/java/org/apache/lucene/index/NormsWriterPerField.java (original)
+++ lucene/java/trunk/src/java/org/apache/lucene/index/NormsWriterPerField.java Thu Jan 21
11:54:50 2010
@@ -63,11 +63,13 @@
   
   @Override
   void finish() {
-    assert docIDs.length == norms.length;
     if (fieldInfo.isIndexed && !fieldInfo.omitNorms) {
       if (docIDs.length <= upto) {
         assert docIDs.length == upto;
         docIDs = ArrayUtil.grow(docIDs, 1+upto);
+      }
+      if (norms.length <= upto) {
+        assert norms.length == upto;
         norms = ArrayUtil.grow(norms, 1+upto);
       }
       final float norm = docState.similarity.computeNorm(fieldInfo.name, fieldState);

Modified: lucene/java/trunk/src/java/org/apache/lucene/index/StoredFieldsWriter.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/index/StoredFieldsWriter.java?rev=901662&r1=901661&r2=901662&view=diff
==============================================================================
--- lucene/java/trunk/src/java/org/apache/lucene/index/StoredFieldsWriter.java (original)
+++ lucene/java/trunk/src/java/org/apache/lucene/index/StoredFieldsWriter.java Thu Jan 21
11:54:50 2010
@@ -20,6 +20,7 @@
 import java.io.IOException;
 import org.apache.lucene.store.RAMOutputStream;
 import org.apache.lucene.util.ArrayUtil;
+import org.apache.lucene.util.RamUsageEstimator;
 
 /** This is a DocFieldConsumer that writes stored fields. */
 final class StoredFieldsWriter {
@@ -108,7 +109,7 @@
         // enough space to recycle all outstanding PerDoc
         // instances
         assert allocCount == 1+docFreeList.length;
-        docFreeList = new PerDoc[ArrayUtil.getNextSize(allocCount)];
+        docFreeList = new PerDoc[ArrayUtil.oversize(allocCount, RamUsageEstimator.NUM_BYTES_OBJECT_REF)];
       }
       return new PerDoc();
     } else

Modified: lucene/java/trunk/src/java/org/apache/lucene/index/TermVectorsTermsWriter.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/index/TermVectorsTermsWriter.java?rev=901662&r1=901661&r2=901662&view=diff
==============================================================================
--- lucene/java/trunk/src/java/org/apache/lucene/index/TermVectorsTermsWriter.java (original)
+++ lucene/java/trunk/src/java/org/apache/lucene/index/TermVectorsTermsWriter.java Thu Jan
21 11:54:50 2010
@@ -20,6 +20,7 @@
 import org.apache.lucene.store.IndexOutput;
 import org.apache.lucene.store.RAMOutputStream;
 import org.apache.lucene.util.ArrayUtil;
+import org.apache.lucene.util.RamUsageEstimator;
 
 import java.io.IOException;
 import java.util.Collection;
@@ -117,7 +118,7 @@
         // enough space to recycle all outstanding PerDoc
         // instances
         assert allocCount == 1+docFreeList.length;
-        docFreeList = new PerDoc[ArrayUtil.getNextSize(allocCount)];
+        docFreeList = new PerDoc[ArrayUtil.oversize(allocCount, RamUsageEstimator.NUM_BYTES_OBJECT_REF)];
       }
       return new PerDoc();
     } else
@@ -266,6 +267,8 @@
     void addField(final int fieldNumber) {
       if (numVectorFields == fieldNumbers.length) {
         fieldNumbers = ArrayUtil.grow(fieldNumbers);
+      }
+      if (numVectorFields == fieldPointers.length) {
         fieldPointers = ArrayUtil.grow(fieldPointers);
       }
       fieldNumbers[numVectorFields] = fieldNumber;

Modified: lucene/java/trunk/src/java/org/apache/lucene/index/TermsHash.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/index/TermsHash.java?rev=901662&r1=901661&r2=901662&view=diff
==============================================================================
--- lucene/java/trunk/src/java/org/apache/lucene/index/TermsHash.java (original)
+++ lucene/java/trunk/src/java/org/apache/lucene/index/TermsHash.java Thu Jan 21 11:54:50
2010
@@ -26,6 +26,7 @@
 import java.io.IOException;
 
 import org.apache.lucene.util.ArrayUtil;
+import org.apache.lucene.util.RamUsageEstimator;
 
 /** This class implements {@link InvertedDocConsumer}, which
  *  is passed each token produced by the analyzer on each
@@ -89,7 +90,7 @@
 
     assert postingsFreeCount == postingsAllocCount: Thread.currentThread().getName() + ":
postingsFreeCount=" + postingsFreeCount + " postingsAllocCount=" + postingsAllocCount + "
consumer=" + consumer;
 
-    final int newSize = ArrayUtil.getShrinkSize(postingsFreeList.length, postingsAllocCount);
+    final int newSize = ArrayUtil.getShrinkSize(postingsFreeList.length, postingsAllocCount,
RamUsageEstimator.NUM_BYTES_OBJECT_REF);
     if (newSize != postingsFreeList.length) {
       RawPostingList[] newArray = new RawPostingList[newSize];
       System.arraycopy(postingsFreeList, 0, newArray, 0, postingsFreeCount);
@@ -222,7 +223,7 @@
       if (newPostingsAllocCount > postingsFreeList.length)
         // Pre-allocate the postingsFreeList so it's large
         // enough to hold all postings we've given out
-        postingsFreeList = new RawPostingList[ArrayUtil.getNextSize(newPostingsAllocCount)];
+        postingsFreeList = new RawPostingList[ArrayUtil.oversize(newPostingsAllocCount, RamUsageEstimator.NUM_BYTES_OBJECT_REF)];
     }
 
     postingsFreeCount -= numToCopy;

Modified: lucene/java/trunk/src/java/org/apache/lucene/util/ArrayUtil.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/util/ArrayUtil.java?rev=901662&r1=901661&r2=901662&view=diff
==============================================================================
--- lucene/java/trunk/src/java/org/apache/lucene/util/ArrayUtil.java (original)
+++ lucene/java/trunk/src/java/org/apache/lucene/util/ArrayUtil.java Thu Jan 21 11:54:50 2010
@@ -7,9 +7,9 @@
  * The ASF licenses this file to You under the Apache License, Version 2.0
  * (the "License"); you may not use this file except in compliance with
  * the License.  You may obtain a copy of the License at
- * <p/>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -122,20 +122,95 @@
  END APACHE HARMONY CODE
   */
 
+  /** Returns an array size >= minTargetSize, generally
+   *  over-allocating exponentially to achieve amortized
+   *  linear-time cost as the array grows.
+   *
+   *  NOTE: this was originally borrowed from Python 2.4.2
+   *  listobject.c sources (attribution in LICENSE.txt), but
+   *  has now been substantially changed based on
+   *  discussions from java-dev thread with subject "Dynamic
+   *  array reallocation algorithms", started on Jan 12
+   *  2010.
+   *
+   * @param minTargetSize Minimum required value to be returned.
+   * @param bytesPerElement Bytes used by each element of
+   * the array.  See constants in {@link RamUsageEstimator}.
+   *
+   * @lucene.internal
+   */
+
+  public static int oversize(int minTargetSize, int bytesPerElement) {
+
+    if (minTargetSize < 0) {
+      // catch usage that accidentally overflows int
+      throw new IllegalArgumentException("invalid array size " + minTargetSize);
+    }
+
+    if (minTargetSize == 0) {
+      // wait until at least one element is requested
+      return 0;
+    }
 
-  public static int getNextSize(int targetSize) {
-    /* This over-allocates proportional to the list size, making room
-     * for additional growth.  The over-allocation is mild, but is
-     * enough to give linear-time amortized behavior over a long
-     * sequence of appends() in the presence of a poorly-performing
-     * system realloc().
-     * The growth pattern is:  0, 4, 8, 16, 25, 35, 46, 58, 72, 88, ...
-     */
-    return (targetSize >> 3) + (targetSize < 9 ? 3 : 6) + targetSize;
+    // asymptotic exponential growth by 1/8th, favors
+    // spending a bit more CPU to not tye up too much wasted
+    // RAM:
+    int extra = minTargetSize >> 3;
+
+    if (extra < 3) {
+      // for very small arrays, where constant overhead of
+      // realloc is presumably relatively high, we grow
+      // faster
+      extra = 3;
+    }
+
+    int newSize = minTargetSize + extra;
+
+    // add 7 to allow for worst case byte alignment addition below:
+    if (newSize+7 < 0) {
+      // int overflowed -- return max allowed array size
+      return Integer.MAX_VALUE;
+    }
+
+    if (Constants.JRE_IS_64BIT) {
+      // round up to 8 byte alignment in 64bit env
+      switch(bytesPerElement) {
+      case 4:
+        // round up to multiple of 2
+        return (newSize + 1) & 0x7ffffffe;
+      case 2:
+        // round up to multiple of 4
+        return (newSize + 3) & 0x7ffffffc;
+      case 1:
+        // round up to multiple of 8
+        return (newSize + 7) & 0x7ffffff8;
+      case 8:
+        // no rounding
+      default:
+        // odd (invalid?) size
+        return newSize;
+      }
+    } else {
+      // round up to 4 byte alignment in 64bit env
+      switch(bytesPerElement) {
+      case 2:
+        // round up to multiple of 2
+        return (newSize + 1) & 0x7ffffffe;
+      case 1:
+        // round up to multiple of 4
+        return (newSize + 3) & 0x7ffffffc;
+      case 4:
+      case 8:
+        // no rounding
+      default:
+        // odd (invalid?) size
+        return newSize;
+      }
+    }
   }
 
-  public static int getShrinkSize(int currentSize, int targetSize) {
-    final int newSize = getNextSize(targetSize);
+  public static int getShrinkSize(int currentSize, int targetSize, int bytesPerElement) {
+    final int newSize = oversize(targetSize, bytesPerElement);
     // Only reallocate if we are "substantially" smaller.
     // This saves us from "running hot" (constantly making a
     // bit bigger then a bit smaller, over and over):
@@ -147,7 +222,7 @@
 
   public static int[] grow(int[] array, int minSize) {
     if (array.length < minSize) {
-      int[] newArray = new int[getNextSize(minSize)];
+      int[] newArray = new int[oversize(minSize, RamUsageEstimator.NUM_BYTES_INT)];
       System.arraycopy(array, 0, newArray, 0, array.length);
       return newArray;
     } else
@@ -159,7 +234,7 @@
   }
 
   public static int[] shrink(int[] array, int targetSize) {
-    final int newSize = getShrinkSize(array.length, targetSize);
+    final int newSize = getShrinkSize(array.length, targetSize, RamUsageEstimator.NUM_BYTES_INT);
     if (newSize != array.length) {
       int[] newArray = new int[newSize];
       System.arraycopy(array, 0, newArray, 0, newSize);
@@ -170,7 +245,7 @@
 
   public static long[] grow(long[] array, int minSize) {
     if (array.length < minSize) {
-      long[] newArray = new long[getNextSize(minSize)];
+      long[] newArray = new long[oversize(minSize, RamUsageEstimator.NUM_BYTES_LONG)];
       System.arraycopy(array, 0, newArray, 0, array.length);
       return newArray;
     } else
@@ -182,7 +257,7 @@
   }
 
   public static long[] shrink(long[] array, int targetSize) {
-    final int newSize = getShrinkSize(array.length, targetSize);
+    final int newSize = getShrinkSize(array.length, targetSize, RamUsageEstimator.NUM_BYTES_LONG);
     if (newSize != array.length) {
       long[] newArray = new long[newSize];
       System.arraycopy(array, 0, newArray, 0, newSize);
@@ -193,7 +268,7 @@
 
   public static byte[] grow(byte[] array, int minSize) {
     if (array.length < minSize) {
-      byte[] newArray = new byte[getNextSize(minSize)];
+      byte[] newArray = new byte[oversize(minSize, 1)];
       System.arraycopy(array, 0, newArray, 0, array.length);
       return newArray;
     } else
@@ -205,7 +280,7 @@
   }
 
   public static byte[] shrink(byte[] array, int targetSize) {
-    final int newSize = getShrinkSize(array.length, targetSize);
+    final int newSize = getShrinkSize(array.length, targetSize, 1);
     if (newSize != array.length) {
       byte[] newArray = new byte[newSize];
       System.arraycopy(array, 0, newArray, 0, newSize);

Modified: lucene/java/trunk/src/java/org/apache/lucene/util/RamUsageEstimator.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/util/RamUsageEstimator.java?rev=901662&r1=901661&r2=901662&view=diff
==============================================================================
--- lucene/java/trunk/src/java/org/apache/lucene/util/RamUsageEstimator.java (original)
+++ lucene/java/trunk/src/java/org/apache/lucene/util/RamUsageEstimator.java Thu Jan 21 11:54:50
2010
@@ -43,6 +43,14 @@
   private int arraySize;
   private int classSize;
 
+  public final static int NUM_BYTES_OBJECT_REF = Constants.JRE_IS_64BIT ? 8 : 4;
+  public final static int NUM_BYTES_CHAR = 2;
+  public final static int NUM_BYTES_SHORT = 2;
+  public final static int NUM_BYTES_INT = 4;
+  public final static int NUM_BYTES_LONG = 8;
+  public final static int NUM_BYTES_FLOAT = 4;
+  public final static int NUM_BYTES_DOUBLE = 8;
+
   private boolean checkInterned;
 
   /**

Modified: lucene/java/trunk/src/test/org/apache/lucene/analysis/TestToken.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/analysis/TestToken.java?rev=901662&r1=901661&r2=901662&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/analysis/TestToken.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/analysis/TestToken.java Thu Jan 21 11:54:50
2010
@@ -85,7 +85,6 @@
       buf.append(buf.toString());
     }
     assertEquals(1048576, t.termLength());
-    assertEquals(1179654, t.termBuffer().length);
 
     // now as a string, first variant
     t = new Token();
@@ -99,7 +98,6 @@
       buf.append(content);
     }
     assertEquals(1048576, t.termLength());
-    assertEquals(1179654, t.termBuffer().length);
 
     // now as a string, second variant
     t = new Token();
@@ -113,7 +111,6 @@
       buf.append(content);
     }
     assertEquals(1048576, t.termLength());
-    assertEquals(1179654, t.termBuffer().length);
 
     // Test for slow growth to a long term
     t = new Token();
@@ -127,7 +124,6 @@
       buf.append("a");
     }
     assertEquals(20000, t.termLength());
-    assertEquals(20167, t.termBuffer().length);
 
     // Test for slow growth to a long term
     t = new Token();
@@ -141,7 +137,6 @@
       buf.append("a");
     }
     assertEquals(20000, t.termLength());
-    assertEquals(20167, t.termBuffer().length);
   }
 
   public void testToString() throws Exception {

Modified: lucene/java/trunk/src/test/org/apache/lucene/analysis/tokenattributes/TestTermAttributeImpl.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/analysis/tokenattributes/TestTermAttributeImpl.java?rev=901662&r1=901661&r2=901662&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/analysis/tokenattributes/TestTermAttributeImpl.java
(original)
+++ lucene/java/trunk/src/test/org/apache/lucene/analysis/tokenattributes/TestTermAttributeImpl.java
Thu Jan 21 11:54:50 2010
@@ -49,7 +49,6 @@
       buf.append(buf.toString());
     }
     assertEquals(1048576, t.termLength());
-    assertEquals(1179654, t.termBuffer().length);
 
     // now as a string, first variant
     t = new TermAttributeImpl();
@@ -63,7 +62,6 @@
       buf.append(content);
     }
     assertEquals(1048576, t.termLength());
-    assertEquals(1179654, t.termBuffer().length);
 
     // now as a string, second variant
     t = new TermAttributeImpl();
@@ -77,7 +75,6 @@
       buf.append(content);
     }
     assertEquals(1048576, t.termLength());
-    assertEquals(1179654, t.termBuffer().length);
 
     // Test for slow growth to a long term
     t = new TermAttributeImpl();
@@ -91,7 +88,6 @@
       buf.append("a");
     }
     assertEquals(20000, t.termLength());
-    assertEquals(20167, t.termBuffer().length);
 
     // Test for slow growth to a long term
     t = new TermAttributeImpl();
@@ -105,7 +101,6 @@
       buf.append("a");
     }
     assertEquals(20000, t.termLength());
-    assertEquals(20167, t.termBuffer().length);
   }
 
   public void testToString() throws Exception {

Added: lucene/java/trunk/src/test/org/apache/lucene/util/TestArrayUtil.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/util/TestArrayUtil.java?rev=901662&view=auto
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/util/TestArrayUtil.java (added)
+++ lucene/java/trunk/src/test/org/apache/lucene/util/TestArrayUtil.java Thu Jan 21 11:54:50
2010
@@ -0,0 +1,59 @@
+package org.apache.lucene.util;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.util.Random;
+
+public class TestArrayUtil extends LuceneTestCase {
+
+  // Ensure ArrayUtil.getNextSize gives linear amortized cost of realloc/copy
+  public void testGrowth() {
+    int currentSize = 0;
+    long copyCost = 0;
+
+    // Make sure ArrayUtil hits Integer.MAX_VALUE, if we insist:
+    while(currentSize != Integer.MAX_VALUE) {
+      int nextSize = ArrayUtil.oversize(1+currentSize, RamUsageEstimator.NUM_BYTES_OBJECT_REF);
+      assertTrue(nextSize > currentSize);
+      if (currentSize > 0) {
+        copyCost += currentSize;
+        double copyCostPerElement = ((double) copyCost)/currentSize;
+        assertTrue("cost " + copyCostPerElement, copyCostPerElement < 10.0);
+      }
+      currentSize = nextSize;
+    }
+  }
+
+  public void testMaxSize() {
+    // intentionally pass invalid elemSizes:
+    for(int elemSize=0;elemSize<10;elemSize++) {
+      assertEquals(Integer.MAX_VALUE, ArrayUtil.oversize(Integer.MAX_VALUE, elemSize));
+      assertEquals(Integer.MAX_VALUE, ArrayUtil.oversize(Integer.MAX_VALUE-1, elemSize));
+    }
+  }
+
+  public void testInvalidElementSizes() {
+    final Random r = newRandom();
+    for(int iter=0;iter<10000;iter++) {
+      final int minTargetSize = r.nextInt(Integer.MAX_VALUE);
+      final int elemSize = r.nextInt(11);
+      final int v = ArrayUtil.oversize(minTargetSize, elemSize);
+      assertTrue(v >= minTargetSize);
+    }
+  }
+}

Propchange: lucene/java/trunk/src/test/org/apache/lucene/util/TestArrayUtil.java
------------------------------------------------------------------------------
    svn:eol-style = native

Modified: lucene/java/trunk/src/test/org/apache/lucene/util/TestIndexableBinaryStringTools.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/util/TestIndexableBinaryStringTools.java?rev=901662&r1=901661&r2=901662&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/util/TestIndexableBinaryStringTools.java
(original)
+++ lucene/java/trunk/src/test/org/apache/lucene/util/TestIndexableBinaryStringTools.java
Thu Jan 21 11:54:50 2010
@@ -164,14 +164,14 @@
       int encodedLen1 = IndexableBinaryStringTools.getEncodedLength(
           originalArray1, 0, numBytes1);
       if (encodedLen1 > encoded1.length)
-        encoded1 = new char[ArrayUtil.getNextSize(encodedLen1)];
+        encoded1 = new char[ArrayUtil.oversize(encodedLen1, RamUsageEstimator.NUM_BYTES_CHAR)];
       IndexableBinaryStringTools.encode(originalArray1, 0, numBytes1, encoded1,
           0, encodedLen1);
 
       int encodedLen2 = IndexableBinaryStringTools.getEncodedLength(original2,
           0, numBytes2);
       if (encodedLen2 > encoded2.length)
-        encoded2 = new char[ArrayUtil.getNextSize(encodedLen2)];
+        encoded2 = new char[ArrayUtil.oversize(encodedLen2, RamUsageEstimator.NUM_BYTES_CHAR)];
       IndexableBinaryStringTools.encode(original2, 0, numBytes2, encoded2, 0,
           encodedLen2);
 
@@ -308,7 +308,7 @@
       int encodedLen = IndexableBinaryStringTools.getEncodedLength(binary, 0,
           numBytes);
       if (encoded.length < encodedLen)
-        encoded = new char[ArrayUtil.getNextSize(encodedLen)];
+        encoded = new char[ArrayUtil.oversize(encodedLen, RamUsageEstimator.NUM_BYTES_CHAR)];
       IndexableBinaryStringTools.encode(binary, 0, numBytes, encoded, 0,
           encodedLen);
 



Mime
View raw message