Return-Path: X-Original-To: apmail-lucene-commits-archive@www.apache.org Delivered-To: apmail-lucene-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id F2D9C17746 for ; Tue, 21 Oct 2014 14:08:56 +0000 (UTC) Received: (qmail 53494 invoked by uid 500); 21 Oct 2014 14:08:56 -0000 Mailing-List: contact commits-help@lucene.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@lucene.apache.org Delivered-To: mailing list commits@lucene.apache.org Received: (qmail 53485 invoked by uid 99); 21 Oct 2014 14:08:56 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 21 Oct 2014 14:08:56 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 21 Oct 2014 14:08:55 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 0A18323889D7; Tue, 21 Oct 2014 14:08:04 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1633385 - in /lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/codecs/lucene50: Lucene50PostingsFormat.java Lucene50PostingsReader.java Lucene50PostingsWriter.java Date: Tue, 21 Oct 2014 14:08:04 -0000 To: commits@lucene.apache.org From: rmuir@apache.org X-Mailer: svnmailer-1.0.9 Message-Id: <20141021140805.0A18323889D7@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: rmuir Date: Tue Oct 21 14:08:04 2014 New Revision: 1633385 URL: http://svn.apache.org/r1633385 Log: LUCENE-5969: clean up constants Modified: lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50PostingsFormat.java lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50PostingsReader.java lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50PostingsWriter.java Modified: lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50PostingsFormat.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50PostingsFormat.java?rev=1633385&r1=1633384&r2=1633385&view=diff ============================================================================== --- lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50PostingsFormat.java (original) +++ lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50PostingsFormat.java Tue Oct 21 14:08:04 2014 @@ -20,6 +20,7 @@ package org.apache.lucene.codecs.lucene5 import java.io.IOException; +import org.apache.lucene.codecs.BlockTermState; import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.codecs.FieldsConsumer; import org.apache.lucene.codecs.FieldsProducer; @@ -33,6 +34,7 @@ import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.FieldInfo.IndexOptions; import org.apache.lucene.index.SegmentReadState; import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.index.TermState; import org.apache.lucene.store.DataOutput; import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.packed.PackedInts; @@ -373,6 +375,21 @@ public final class Lucene50PostingsForma * See chapter: Payloads and Offsets */ public static final String PAY_EXTENSION = "pay"; + + /** + * Expert: The maximum number of skip levels. Smaller values result in + * slightly smaller indexes, but slower skipping in big posting lists. + */ + static final int MAX_SKIP_LEVELS = 10; + + final static String TERMS_CODEC = "Lucene50PostingsWriterTerms"; + final static String DOC_CODEC = "Lucene50PostingsWriterDoc"; + final static String POS_CODEC = "Lucene50PostingsWriterPos"; + final static String PAY_CODEC = "Lucene50PostingsWriterPay"; + + // Increment version to change it + final static int VERSION_START = 0; + final static int VERSION_CURRENT = VERSION_START; private final int minTermBlockSize; private final int maxTermBlockSize; @@ -440,4 +457,39 @@ public final class Lucene50PostingsForma } } } + + final static class IntBlockTermState extends BlockTermState { + long docStartFP = 0; + long posStartFP = 0; + long payStartFP = 0; + long skipOffset = -1; + long lastPosBlockOffset = -1; + // docid when there is a single pulsed posting, otherwise -1 + // freq is always implicitly totalTermFreq in this case. + int singletonDocID = -1; + + @Override + public IntBlockTermState clone() { + IntBlockTermState other = new IntBlockTermState(); + other.copyFrom(this); + return other; + } + + @Override + public void copyFrom(TermState _other) { + super.copyFrom(_other); + IntBlockTermState other = (IntBlockTermState) _other; + docStartFP = other.docStartFP; + posStartFP = other.posStartFP; + payStartFP = other.payStartFP; + lastPosBlockOffset = other.lastPosBlockOffset; + skipOffset = other.skipOffset; + singletonDocID = other.singletonDocID; + } + + @Override + public String toString() { + return super.toString() + " docStartFP=" + docStartFP + " posStartFP=" + posStartFP + " payStartFP=" + payStartFP + " lastPosBlockOffset=" + lastPosBlockOffset + " singletonDocID=" + singletonDocID; + } + } } Modified: lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50PostingsReader.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50PostingsReader.java?rev=1633385&r1=1633384&r2=1633385&view=diff ============================================================================== --- lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50PostingsReader.java (original) +++ lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50PostingsReader.java Tue Oct 21 14:08:04 2014 @@ -17,10 +17,16 @@ package org.apache.lucene.codecs.lucene5 * limitations under the License. */ -import static org.apache.lucene.codecs.lucene50.Lucene50PostingsFormat.BLOCK_SIZE; import static org.apache.lucene.codecs.lucene50.ForUtil.MAX_DATA_SIZE; import static org.apache.lucene.codecs.lucene50.ForUtil.MAX_ENCODED_SIZE; -import static org.apache.lucene.codecs.lucene50.Lucene50PostingsWriter.IntBlockTermState; +import static org.apache.lucene.codecs.lucene50.Lucene50PostingsFormat.BLOCK_SIZE; +import static org.apache.lucene.codecs.lucene50.Lucene50PostingsFormat.DOC_CODEC; +import static org.apache.lucene.codecs.lucene50.Lucene50PostingsFormat.MAX_SKIP_LEVELS; +import static org.apache.lucene.codecs.lucene50.Lucene50PostingsFormat.PAY_CODEC; +import static org.apache.lucene.codecs.lucene50.Lucene50PostingsFormat.POS_CODEC; +import static org.apache.lucene.codecs.lucene50.Lucene50PostingsFormat.TERMS_CODEC; +import static org.apache.lucene.codecs.lucene50.Lucene50PostingsFormat.VERSION_CURRENT; +import static org.apache.lucene.codecs.lucene50.Lucene50PostingsFormat.VERSION_START; import java.io.IOException; import java.util.Arrays; @@ -29,6 +35,7 @@ import java.util.Collections; import org.apache.lucene.codecs.BlockTermState; import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.codecs.PostingsReaderBase; +import org.apache.lucene.codecs.lucene50.Lucene50PostingsFormat.IntBlockTermState; import org.apache.lucene.index.DocsAndPositionsEnum; import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.FieldInfo; @@ -48,7 +55,6 @@ import org.apache.lucene.util.RamUsageEs * Concrete class that reads docId(maybe frq,pos,offset,payloads) list * with postings format. * - * @see Lucene50SkipReader for details * @lucene.experimental */ public final class Lucene50PostingsReader extends PostingsReaderBase { @@ -68,42 +74,29 @@ public final class Lucene50PostingsReade IndexInput docIn = null; IndexInput posIn = null; IndexInput payIn = null; + + // NOTE: these data files are too costly to verify checksum against all the bytes on open, + // but for now we at least verify proper structure of the checksum footer: which looks + // for FOOTER_MAGIC + algorithmID. This is cheap and can detect some forms of corruption + // such as file truncation. + String docName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, Lucene50PostingsFormat.DOC_EXTENSION); try { docIn = state.directory.openInput(docName, state.context); - version = CodecUtil.checkSegmentHeader(docIn, - Lucene50PostingsWriter.DOC_CODEC, - Lucene50PostingsWriter.VERSION_START, - Lucene50PostingsWriter.VERSION_CURRENT, - state.segmentInfo.getId(), state.segmentSuffix); + version = CodecUtil.checkSegmentHeader(docIn, DOC_CODEC, VERSION_START, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix); forUtil = new ForUtil(docIn); - - // NOTE: data file is too costly to verify checksum against all the bytes on open, - // but for now we at least verify proper structure of the checksum footer: which looks - // for FOOTER_MAGIC + algorithmID. This is cheap and can detect some forms of corruption - // such as file truncation. CodecUtil.retrieveChecksum(docIn); if (state.fieldInfos.hasProx()) { String proxName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, Lucene50PostingsFormat.POS_EXTENSION); posIn = state.directory.openInput(proxName, state.context); - CodecUtil.checkSegmentHeader(posIn, Lucene50PostingsWriter.POS_CODEC, version, version, state.segmentInfo.getId(), state.segmentSuffix); - - // NOTE: data file is too costly to verify checksum against all the bytes on open, - // but for now we at least verify proper structure of the checksum footer: which looks - // for FOOTER_MAGIC + algorithmID. This is cheap and can detect some forms of corruption - // such as file truncation. + CodecUtil.checkSegmentHeader(posIn, POS_CODEC, version, version, state.segmentInfo.getId(), state.segmentSuffix); CodecUtil.retrieveChecksum(posIn); if (state.fieldInfos.hasPayloads() || state.fieldInfos.hasOffsets()) { String payName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, Lucene50PostingsFormat.PAY_EXTENSION); payIn = state.directory.openInput(payName, state.context); - CodecUtil.checkSegmentHeader(payIn, Lucene50PostingsWriter.PAY_CODEC, version, version, state.segmentInfo.getId(), state.segmentSuffix); - - // NOTE: data file is too costly to verify checksum against all the bytes on open, - // but for now we at least verify proper structure of the checksum footer: which looks - // for FOOTER_MAGIC + algorithmID. This is cheap and can detect some forms of corruption - // such as file truncation. + CodecUtil.checkSegmentHeader(payIn, PAY_CODEC, version, version, state.segmentInfo.getId(), state.segmentSuffix); CodecUtil.retrieveChecksum(payIn); } } @@ -122,12 +115,7 @@ public final class Lucene50PostingsReade @Override public void init(IndexInput termsIn, SegmentReadState state) throws IOException { // Make sure we are talking to the matching postings writer - CodecUtil.checkSegmentHeader(termsIn, - Lucene50PostingsWriter.TERMS_CODEC, - Lucene50PostingsWriter.VERSION_START, - Lucene50PostingsWriter.VERSION_CURRENT, - state.segmentInfo.getId(), - state.segmentSuffix); + CodecUtil.checkSegmentHeader(termsIn, TERMS_CODEC, VERSION_START, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix); final int indexBlockSize = termsIn.readVInt(); if (indexBlockSize != BLOCK_SIZE) { throw new IllegalStateException("index-time BLOCK_SIZE (" + indexBlockSize + ") != read-time BLOCK_SIZE (" + BLOCK_SIZE + ")"); @@ -414,11 +402,11 @@ public final class Lucene50PostingsReade if (skipper == null) { // Lazy init: first time this enum has ever been used for skipping skipper = new Lucene50SkipReader(docIn.clone(), - Lucene50PostingsWriter.MAX_SKIP_LEVELS, - BLOCK_SIZE, - indexHasPos, - indexHasOffsets, - indexHasPayloads); + MAX_SKIP_LEVELS, + BLOCK_SIZE, + indexHasPos, + indexHasOffsets, + indexHasPayloads); } if (!skipped) { @@ -692,11 +680,11 @@ public final class Lucene50PostingsReade if (skipper == null) { // Lazy init: first time this enum has ever been used for skipping skipper = new Lucene50SkipReader(docIn.clone(), - Lucene50PostingsWriter.MAX_SKIP_LEVELS, - BLOCK_SIZE, - true, - indexHasOffsets, - indexHasPayloads); + MAX_SKIP_LEVELS, + BLOCK_SIZE, + true, + indexHasOffsets, + indexHasPayloads); } if (!skipped) { @@ -1118,7 +1106,7 @@ public final class Lucene50PostingsReade if (skipper == null) { // Lazy init: first time this enum has ever been used for skipping skipper = new Lucene50SkipReader(docIn.clone(), - Lucene50PostingsWriter.MAX_SKIP_LEVELS, + MAX_SKIP_LEVELS, BLOCK_SIZE, true, indexHasOffsets, Modified: lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50PostingsWriter.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50PostingsWriter.java?rev=1633385&r1=1633384&r2=1633385&view=diff ============================================================================== --- lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50PostingsWriter.java (original) +++ lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50PostingsWriter.java Tue Oct 21 14:08:04 2014 @@ -17,11 +17,22 @@ package org.apache.lucene.codecs.lucene5 * limitations under the License. */ +import static org.apache.lucene.codecs.lucene50.ForUtil.MAX_DATA_SIZE; +import static org.apache.lucene.codecs.lucene50.ForUtil.MAX_ENCODED_SIZE; +import static org.apache.lucene.codecs.lucene50.Lucene50PostingsFormat.BLOCK_SIZE; +import static org.apache.lucene.codecs.lucene50.Lucene50PostingsFormat.DOC_CODEC; +import static org.apache.lucene.codecs.lucene50.Lucene50PostingsFormat.MAX_SKIP_LEVELS; +import static org.apache.lucene.codecs.lucene50.Lucene50PostingsFormat.PAY_CODEC; +import static org.apache.lucene.codecs.lucene50.Lucene50PostingsFormat.POS_CODEC; +import static org.apache.lucene.codecs.lucene50.Lucene50PostingsFormat.TERMS_CODEC; +import static org.apache.lucene.codecs.lucene50.Lucene50PostingsFormat.VERSION_CURRENT; + import java.io.IOException; import org.apache.lucene.codecs.BlockTermState; import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.codecs.PushPostingsWriterBase; +import org.apache.lucene.codecs.lucene50.Lucene50PostingsFormat.IntBlockTermState; import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.IndexFileNames; @@ -34,11 +45,6 @@ import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.packed.PackedInts; -import static org.apache.lucene.codecs.lucene50.ForUtil.MAX_DATA_SIZE; -import static org.apache.lucene.codecs.lucene50.ForUtil.MAX_ENCODED_SIZE; -import static org.apache.lucene.codecs.lucene50.Lucene50PostingsFormat.BLOCK_SIZE; - - /** * Concrete class that writes docId(maybe frq,pos,offset,payloads) list * with postings format. @@ -50,21 +56,6 @@ import static org.apache.lucene.codecs.l */ public final class Lucene50PostingsWriter extends PushPostingsWriterBase { - /** - * Expert: The maximum number of skip levels. Smaller values result in - * slightly smaller indexes, but slower skipping in big posting lists. - */ - static final int MAX_SKIP_LEVELS = 10; - - final static String TERMS_CODEC = "Lucene50PostingsWriterTerms"; - final static String DOC_CODEC = "Lucene50PostingsWriterDoc"; - final static String POS_CODEC = "Lucene50PostingsWriterPos"; - final static String PAY_CODEC = "Lucene50PostingsWriterPay"; - - // Increment version to change it - final static int VERSION_START = 0; - final static int VERSION_CURRENT = VERSION_START; - IndexOutput docOut; IndexOutput posOut; IndexOutput payOut; @@ -178,41 +169,6 @@ public final class Lucene50PostingsWrite encoded = new byte[MAX_ENCODED_SIZE]; } - final static class IntBlockTermState extends BlockTermState { - long docStartFP = 0; - long posStartFP = 0; - long payStartFP = 0; - long skipOffset = -1; - long lastPosBlockOffset = -1; - // docid when there is a single pulsed posting, otherwise -1 - // freq is always implicitly totalTermFreq in this case. - int singletonDocID = -1; - - @Override - public IntBlockTermState clone() { - IntBlockTermState other = new IntBlockTermState(); - other.copyFrom(this); - return other; - } - - @Override - public void copyFrom(TermState _other) { - super.copyFrom(_other); - IntBlockTermState other = (IntBlockTermState) _other; - docStartFP = other.docStartFP; - posStartFP = other.posStartFP; - payStartFP = other.payStartFP; - lastPosBlockOffset = other.lastPosBlockOffset; - skipOffset = other.skipOffset; - singletonDocID = other.singletonDocID; - } - - @Override - public String toString() { - return super.toString() + " docStartFP=" + docStartFP + " posStartFP=" + posStartFP + " payStartFP=" + payStartFP + " lastPosBlockOffset=" + lastPosBlockOffset + " singletonDocID=" + singletonDocID; - } - } - @Override public IntBlockTermState newTermState() { return new IntBlockTermState();