Return-Path: Delivered-To: apmail-lucene-java-commits-archive@www.apache.org Received: (qmail 88155 invoked from network); 17 May 2008 16:49:48 -0000 Received: from hermes.apache.org (HELO mail.apache.org) (140.211.11.2) by minotaur.apache.org with SMTP; 17 May 2008 16:49:48 -0000 Received: (qmail 47651 invoked by uid 500); 17 May 2008 16:49:50 -0000 Delivered-To: apmail-lucene-java-commits-archive@lucene.apache.org Received: (qmail 47623 invoked by uid 500); 17 May 2008 16:49:49 -0000 Mailing-List: contact java-commits-help@lucene.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: java-dev@lucene.apache.org Delivered-To: mailing list java-commits@lucene.apache.org Received: (qmail 47614 invoked by uid 99); 17 May 2008 16:49:49 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 17 May 2008 09:49:49 -0700 X-ASF-Spam-Status: No, hits=-2000.0 required=10.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 17 May 2008 16:49:03 +0000 Received: by eris.apache.org (Postfix, from userid 65534) id EA9122388A2A; Sat, 17 May 2008 09:49:18 -0700 (PDT) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r657397 - in /lucene/java/trunk/src: java/org/apache/lucene/index/ test/org/apache/lucene/index/ Date: Sat, 17 May 2008 16:49:18 -0000 To: java-commits@lucene.apache.org From: mikemccand@apache.org X-Mailer: svnmailer-1.0.8 Message-Id: <20080517164918.EA9122388A2A@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: mikemccand Date: Sat May 17 09:49:18 2008 New Revision: 657397 URL: http://svn.apache.org/viewvc?rev=657397&view=rev Log: LUCENE-1283: factor out ByteSliceWriter from DocumentsWriter Added: lucene/java/trunk/src/java/org/apache/lucene/index/ByteSliceWriter.java (with props) lucene/java/trunk/src/test/org/apache/lucene/index/TestByteSlices.java (with props) Modified: lucene/java/trunk/src/java/org/apache/lucene/index/ByteBlockPool.java lucene/java/trunk/src/java/org/apache/lucene/index/DocumentsWriter.java lucene/java/trunk/src/java/org/apache/lucene/index/DocumentsWriterFieldData.java lucene/java/trunk/src/java/org/apache/lucene/index/DocumentsWriterThreadState.java lucene/java/trunk/src/test/org/apache/lucene/index/TestStressIndexing2.java Modified: lucene/java/trunk/src/java/org/apache/lucene/index/ByteBlockPool.java URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/index/ByteBlockPool.java?rev=657397&r1=657396&r2=657397&view=diff ============================================================================== --- lucene/java/trunk/src/java/org/apache/lucene/index/ByteBlockPool.java (original) +++ lucene/java/trunk/src/java/org/apache/lucene/index/ByteBlockPool.java Sat May 17 09:49:18 2008 @@ -37,6 +37,11 @@ final class ByteBlockPool { + abstract static class Allocator { + abstract void recycleByteBlocks(byte[][] blocks, int start, int end); + abstract byte[] getByteBlock(boolean trackAllocations); + } + public byte[][] buffers = new byte[10][]; int bufferUpto = -1; // Which buffer we are upto @@ -45,11 +50,11 @@ public byte[] buffer; // Current head buffer public int byteOffset = -DocumentsWriter.BYTE_BLOCK_SIZE; // Current head offset - private boolean trackAllocations; - DocumentsWriter docWriter; + private final boolean trackAllocations; + private final Allocator allocator; - public ByteBlockPool(DocumentsWriter docWriter, boolean trackAllocations) { - this.docWriter = docWriter; + public ByteBlockPool(Allocator allocator, boolean trackAllocations) { + this.allocator = allocator; this.trackAllocations = trackAllocations; } @@ -66,7 +71,7 @@ if (bufferUpto > 0) // Recycle all but the first buffer - docWriter.recycleByteBlocks(buffers, 1, 1+bufferUpto); + allocator.recycleByteBlocks(buffers, 1, 1+bufferUpto); // Re-use the first buffer bufferUpto = 0; @@ -82,7 +87,7 @@ System.arraycopy(buffers, 0, newBuffers, 0, buffers.length); buffers = newBuffers; } - buffer = buffers[1+bufferUpto] = docWriter.getByteBlock(trackAllocations); + buffer = buffers[1+bufferUpto] = allocator.getByteBlock(trackAllocations); bufferUpto++; byteUpto = 0; Added: lucene/java/trunk/src/java/org/apache/lucene/index/ByteSliceWriter.java URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/index/ByteSliceWriter.java?rev=657397&view=auto ============================================================================== --- lucene/java/trunk/src/java/org/apache/lucene/index/ByteSliceWriter.java (added) +++ lucene/java/trunk/src/java/org/apache/lucene/index/ByteSliceWriter.java Sat May 17 09:49:18 2008 @@ -0,0 +1,89 @@ +package org.apache.lucene.index; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +/** + * Class to write byte streams into slices of shared + * byte[]. This is used by DocumentsWriter to hold the + * posting list for many terms in RAM. + */ + +final class ByteSliceWriter { + + private byte[] slice; + private int upto; + private final ByteBlockPool pool; + + int offset0; + + public ByteSliceWriter(ByteBlockPool pool) { + this.pool = pool; + } + + /** + * Set up the writer to write at address. + */ + public void init(int address) { + slice = pool.buffers[address >> DocumentsWriter.BYTE_BLOCK_SHIFT]; + assert slice != null; + upto = address & DocumentsWriter.BYTE_BLOCK_MASK; + offset0 = address; + assert upto < slice.length; + } + + /** Write byte into byte slice stream */ + public void writeByte(byte b) { + assert slice != null; + if (slice[upto] != 0) { + upto = pool.allocSlice(slice, upto); + slice = pool.buffer; + offset0 = pool.byteOffset; + assert slice != null; + } + slice[upto++] = b; + assert upto != slice.length; + } + + public void writeBytes(final byte[] b, int offset, final int len) { + final int offsetEnd = offset + len; + while(offset < offsetEnd) { + if (slice[upto] != 0) { + // End marker + upto = pool.allocSlice(slice, upto); + slice = pool.buffer; + offset0 = pool.byteOffset; + } + + slice[upto++] = b[offset++]; + assert upto != slice.length; + } + } + + public int getAddress() { + return upto + (offset0 & DocumentsWriter.BYTE_BLOCK_NOT_MASK); + } + + public void writeVInt(int i) { + while ((i & ~0x7F) != 0) { + writeByte((byte)((i & 0x7f) | 0x80)); + i >>>= 7; + } + writeByte((byte) i); + } +} Propchange: lucene/java/trunk/src/java/org/apache/lucene/index/ByteSliceWriter.java ------------------------------------------------------------------------------ svn:eol-style = native Modified: lucene/java/trunk/src/java/org/apache/lucene/index/DocumentsWriter.java URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/index/DocumentsWriter.java?rev=657397&r1=657396&r2=657397&view=diff ============================================================================== --- lucene/java/trunk/src/java/org/apache/lucene/index/DocumentsWriter.java (original) +++ lucene/java/trunk/src/java/org/apache/lucene/index/DocumentsWriter.java Sat May 17 09:49:18 2008 @@ -1473,30 +1473,39 @@ final static int BYTE_BLOCK_MASK = BYTE_BLOCK_SIZE - 1; final static int BYTE_BLOCK_NOT_MASK = ~BYTE_BLOCK_MASK; - private ArrayList freeByteBlocks = new ArrayList(); + private class ByteBlockAllocator extends ByteBlockPool.Allocator { - /* Allocate another byte[] from the shared pool */ - synchronized byte[] getByteBlock(boolean trackAllocations) { - final int size = freeByteBlocks.size(); - final byte[] b; - if (0 == size) { - numBytesAlloc += BYTE_BLOCK_SIZE; - balanceRAM(); - b = new byte[BYTE_BLOCK_SIZE]; - } else - b = (byte[]) freeByteBlocks.remove(size-1); - if (trackAllocations) - numBytesUsed += BYTE_BLOCK_SIZE; - assert numBytesUsed <= numBytesAlloc; - return b; - } + ArrayList freeByteBlocks = new ArrayList(); + + /* Allocate another byte[] from the shared pool */ + byte[] getByteBlock(boolean trackAllocations) { + synchronized(DocumentsWriter.this) { + final int size = freeByteBlocks.size(); + final byte[] b; + if (0 == size) { + numBytesAlloc += BYTE_BLOCK_SIZE; + balanceRAM(); + b = new byte[BYTE_BLOCK_SIZE]; + } else + b = (byte[]) freeByteBlocks.remove(size-1); + if (trackAllocations) + numBytesUsed += BYTE_BLOCK_SIZE; + assert numBytesUsed <= numBytesAlloc; + return b; + } + } - /* Return byte[]'s to the pool */ - synchronized void recycleByteBlocks(byte[][] blocks, int start, int end) { - for(int i=start;i