Return-Path: X-Original-To: apmail-lucene-commits-archive@www.apache.org Delivered-To: apmail-lucene-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 8EED59F78 for ; Sun, 20 Nov 2011 23:46:29 +0000 (UTC) Received: (qmail 51759 invoked by uid 500); 20 Nov 2011 23:46:29 -0000 Mailing-List: contact commits-help@lucene.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@lucene.apache.org Delivered-To: mailing list commits@lucene.apache.org Received: (qmail 51752 invoked by uid 99); 20 Nov 2011 23:46:29 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Sun, 20 Nov 2011 23:46:29 +0000 X-ASF-Spam-Status: No, hits=-1997.8 required=5.0 tests=ALL_TRUSTED,FILL_THIS_FORM,FILL_THIS_FORM_LOAN X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Sun, 20 Nov 2011 23:46:12 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 407592388A02; Sun, 20 Nov 2011 23:45:49 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1204297 [4/10] - in /lucene/dev/branches/solrcloud: ./ dev-tools/eclipse/ dev-tools/idea/lucene/contrib/ dev-tools/idea/lucene/contrib/instantiated/ dev-tools/maven/lucene/contrib/ dev-tools/maven/lucene/contrib/instantiated/ lucene/ lucen... Date: Sun, 20 Nov 2011 23:45:35 -0000 To: commits@lucene.apache.org From: markrmiller@apache.org X-Mailer: svnmailer-1.0.8-patched Message-Id: <20111120234549.407592388A02@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/BlockTreeTermsReader.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/BlockTreeTermsReader.java?rev=1204297&r1=1204296&r2=1204297&view=diff ============================================================================== --- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/BlockTreeTermsReader.java (original) +++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/BlockTreeTermsReader.java Sun Nov 20 23:45:25 2011 @@ -18,12 +18,8 @@ package org.apache.lucene.index.codecs; */ import java.io.ByteArrayOutputStream; -import java.io.Closeable; -import java.io.FileOutputStream; import java.io.IOException; -import java.io.OutputStreamWriter; import java.io.PrintStream; -import java.io.Writer; import java.util.Collection; import java.util.Comparator; import java.util.Iterator; @@ -197,9 +193,6 @@ public class BlockTreeTermsReader extend try { IOUtils.close(in, postingsReader); } finally { - for(FieldReader field : fields.values()) { - field.close(); - } // Clear so refs to terms index is GCable even if // app hangs onto us: fields.clear(); @@ -221,6 +214,11 @@ public class BlockTreeTermsReader extend return fields.get(field); } + @Override + public int getUniqueFieldCount() { + return fields.size(); + } + // Iterates through all fields private class TermFieldsEnum extends FieldsEnum { final Iterator it; @@ -242,8 +240,8 @@ public class BlockTreeTermsReader extend } @Override - public TermsEnum terms() throws IOException { - return current.iterator(); + public Terms terms() throws IOException { + return current; } } @@ -390,7 +388,7 @@ public class BlockTreeTermsReader extend final Outputs fstOutputs = ByteSequenceOutputs.getSingleton(); final BytesRef NO_OUTPUT = fstOutputs.getNoOutput(); - public final class FieldReader extends Terms implements Closeable { + public final class FieldReader extends Terms { final long numTerms; final FieldInfo fieldInfo; final long sumTotalTermFreq; @@ -449,12 +447,7 @@ public class BlockTreeTermsReader extend } @Override - public void close() { - super.close(); - } - - @Override - public TermsEnum iterator() throws IOException { + public TermsEnum iterator(TermsEnum reuse) throws IOException { return new SegmentTermsEnum(); } @@ -742,7 +735,7 @@ public class BlockTreeTermsReader extend } } - private final BytesRef savedStartTerm; + private BytesRef savedStartTerm; // TODO: in some cases we can filter by length? eg // regexp foo*bar must be at least length 6 bytes @@ -782,7 +775,7 @@ public class BlockTreeTermsReader extend f.load(rootCode); // for assert: - savedStartTerm = startTerm == null ? null : new BytesRef(startTerm); + assert setSavedStartTerm(startTerm); currentFrame = f; if (startTerm != null) { @@ -790,6 +783,12 @@ public class BlockTreeTermsReader extend } } + // only for assert: + private boolean setSavedStartTerm(BytesRef startTerm) { + savedStartTerm = startTerm == null ? null : new BytesRef(startTerm); + return true; + } + @Override public TermState termState() throws IOException { currentFrame.decodeMetaData(); @@ -914,8 +913,6 @@ public class BlockTreeTermsReader extend for(int idx=0;idx<=target.length;idx++) { - boolean lastIsSubBlock = false; - while (true) { final int savePos = currentFrame.suffixesReader.getPosition(); final int saveStartBytePos = currentFrame.startBytePos; @@ -950,7 +947,6 @@ public class BlockTreeTermsReader extend return; } } - lastIsSubBlock = isSubBlock; continue; } else if (cmp == 0) { //if (DEBUG) System.out.println(" return term=" + brToString(term)); @@ -1164,7 +1160,7 @@ public class BlockTreeTermsReader extend // Iterates through terms in this field private final class SegmentTermsEnum extends TermsEnum { - private final IndexInput in; + private IndexInput in; private Frame[] stack; private final Frame staticFrame; @@ -1183,29 +1179,21 @@ public class BlockTreeTermsReader extend final BytesRef term = new BytesRef(); - @SuppressWarnings("unchecked") private FST.Arc[] arcs = new FST.Arc[5]; + @SuppressWarnings("unchecked") private FST.Arc[] arcs = new FST.Arc[1]; public SegmentTermsEnum() throws IOException { //if (DEBUG) System.out.println("BTTR.init seg=" + segment); - in = (IndexInput) BlockTreeTermsReader.this.in.clone(); - stack = new Frame[5]; - for(int stackOrd=0;stackOrd(); } - // Init w/ root block; don't use index since it may - // not (and need not) have been loaded - //final FST.Arc arc = index.getFirstArc(arcs[0]); - - // Empty string prefix must have an output in the index! - //assert arc.isFinal(); - currentFrame = staticFrame; final FST.Arc arc; if (index != null) { @@ -1215,8 +1203,9 @@ public class BlockTreeTermsReader extend } else { arc = null; } - currentFrame = pushFrame(arc, rootCode, 0); - currentFrame.loadBlock(); + currentFrame = staticFrame; + //currentFrame = pushFrame(arc, rootCode, 0); + //currentFrame.loadBlock(); validIndexPrefix = 0; // if (DEBUG) { // System.out.println("init frame state " + currentFrame.ord); @@ -1227,6 +1216,12 @@ public class BlockTreeTermsReader extend // computeBlockStats().print(System.out); } + private void initIndexInput() { + if (this.in == null) { + this.in = (IndexInput) BlockTreeTermsReader.this.in.clone(); + } + } + /** Runs next() through the entire terms dict, * computing aggregate statistics. */ public Stats computeBlockStats() throws IOException { @@ -1976,6 +1971,20 @@ public class BlockTreeTermsReader extend @Override public BytesRef next() throws IOException { + if (in == null) { + // Fresh TermsEnum; seek to first term: + final FST.Arc arc; + if (index != null) { + arc = index.getFirstArc(arcs[0]); + // Empty string prefix must have an output in the index! + assert arc.isFinal(); + } else { + arc = null; + } + currentFrame = pushFrame(arc, rootCode, 0); + currentFrame.loadBlock(); + } + targetBeforeCurrentLength = currentFrame.ord; assert !eof; @@ -2243,6 +2252,11 @@ public class BlockTreeTermsReader extend use. */ void loadBlock() throws IOException { + // Clone the IndexInput lazily, so that consumers + // that just pull a TermsEnum to + // seekExact(TermState) don't pay this cost: + initIndexInput(); + if (nextEnt != -1) { // Already loaded return; Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/Codec.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/Codec.java?rev=1204297&r1=1204296&r2=1204297&view=diff ============================================================================== --- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/Codec.java (original) +++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/Codec.java Sun Nov 20 23:45:25 2011 @@ -46,6 +46,10 @@ public abstract class Codec implements N public void files(Directory dir, SegmentInfo info, Set files) throws IOException { postingsFormat().files(dir, info, "", files); storedFieldsFormat().files(dir, info, files); + termVectorsFormat().files(dir, info, files); + fieldInfosFormat().files(dir, info, files); + // TODO: segmentInfosFormat should be allowed to declare additional files + // if it wants, in addition to segments_N docValuesFormat().files(dir, info, files); } @@ -58,6 +62,12 @@ public abstract class Codec implements N /** Encodes/decodes stored fields */ public abstract StoredFieldsFormat storedFieldsFormat(); + /** Encodes/decodes term vectors */ + public abstract TermVectorsFormat termVectorsFormat(); + + /** Encodes/decodes field infos file */ + public abstract FieldInfosFormat fieldInfosFormat(); + /** Encodes/decodes segments file */ public abstract SegmentInfosFormat segmentInfosFormat(); Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosReader.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosReader.java?rev=1204297&r1=1204296&r2=1204297&view=diff ============================================================================== --- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosReader.java (original) +++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosReader.java Sun Nov 20 23:45:25 2011 @@ -18,6 +18,8 @@ package org.apache.lucene.index.codecs; */ import java.io.IOException; +import java.util.HashMap; +import java.util.Map; import org.apache.lucene.index.IndexFileNames; import org.apache.lucene.index.IndexFormatTooOldException; @@ -51,7 +53,7 @@ public class DefaultSegmentInfosReader e infos.setGlobalFieldMapVersion(input.readLong()); } for (int i = input.readInt(); i > 0; i--) { // read segmentInfos - SegmentInfo si = new SegmentInfo(directory, format, input); + SegmentInfo si = readSegmentInfo(directory, format, input); if (si.getVersion() == null) { // Could be a 3.0 - try to open the doc stores - if it fails, it's a // 2.x segment, and an IndexFormatTooOldException will be thrown, @@ -90,4 +92,100 @@ public class DefaultSegmentInfosReader e infos.userData = input.readStringStringMap(); } + + // if we make a preflex impl we can remove a lot of this hair... + public SegmentInfo readSegmentInfo(Directory dir, int format, ChecksumIndexInput input) throws IOException { + final String version; + if (format <= DefaultSegmentInfosWriter.FORMAT_3_1) { + version = input.readString(); + } else { + version = null; + } + final String name = input.readString(); + final int docCount = input.readInt(); + final long delGen = input.readLong(); + final int docStoreOffset = input.readInt(); + final String docStoreSegment; + final boolean docStoreIsCompoundFile; + if (docStoreOffset != -1) { + docStoreSegment = input.readString(); + docStoreIsCompoundFile = input.readByte() == SegmentInfo.YES; + } else { + docStoreSegment = name; + docStoreIsCompoundFile = false; + } + + if (format > DefaultSegmentInfosWriter.FORMAT_4_0) { + // pre-4.0 indexes write a byte if there is a single norms file + byte b = input.readByte(); + assert 1 == b; + } + + final int numNormGen = input.readInt(); + final Map normGen; + if (numNormGen == SegmentInfo.NO) { + normGen = null; + } else { + normGen = new HashMap(); + for(int j=0;j diagnostics = input.readStringStringMap(); + + final int hasVectors; + if (format <= DefaultSegmentInfosWriter.FORMAT_HAS_VECTORS) { + hasVectors = input.readByte(); + } else { + final String storesSegment; + final String ext; + final boolean storeIsCompoundFile; + if (docStoreOffset != -1) { + storesSegment = docStoreSegment; + storeIsCompoundFile = docStoreIsCompoundFile; + ext = IndexFileNames.COMPOUND_FILE_STORE_EXTENSION; + } else { + storesSegment = name; + storeIsCompoundFile = isCompoundFile; + ext = IndexFileNames.COMPOUND_FILE_EXTENSION; + } + final Directory dirToTest; + if (storeIsCompoundFile) { + dirToTest = new CompoundFileDirectory(dir, IndexFileNames.segmentFileName(storesSegment, "", ext), IOContext.READONCE, false); + } else { + dirToTest = dir; + } + try { + // TODO: remove this manual file check or push to preflex codec + hasVectors = dirToTest.fileExists(IndexFileNames.segmentFileName(storesSegment, "", DefaultTermVectorsReader.VECTORS_INDEX_EXTENSION)) ? SegmentInfo.YES : SegmentInfo.NO; + } finally { + if (isCompoundFile) { + dirToTest.close(); + } + } + } + + return new SegmentInfo(dir, version, name, docCount, delGen, docStoreOffset, + docStoreSegment, docStoreIsCompoundFile, normGen, isCompoundFile, + delCount, hasProx, codec, diagnostics, hasVectors); + } } Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosWriter.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosWriter.java?rev=1204297&r1=1204296&r2=1204297&view=diff ============================================================================== --- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosWriter.java (original) +++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosWriter.java Sun Nov 20 23:45:25 2011 @@ -18,6 +18,8 @@ package org.apache.lucene.index.codecs; */ import java.io.IOException; +import java.util.Map; +import java.util.Map.Entry; import org.apache.lucene.index.SegmentInfo; import org.apache.lucene.index.SegmentInfos; @@ -50,6 +52,7 @@ public class DefaultSegmentInfosWriter e /** This must always point to the most recent file format. * whenever you add a new format, make it 1 smaller (negative version logic)! */ + // TODO: move this, as its currently part of required preamble public static final int FORMAT_CURRENT = FORMAT_4_0; /** This must always point to the first supported file format. */ @@ -68,7 +71,7 @@ public class DefaultSegmentInfosWriter e out.writeLong(infos.getGlobalFieldMapVersion()); out.writeInt(infos.size()); // write infos for (SegmentInfo si : infos) { - si.write(out); + writeInfo(out, si); } out.writeStringStringMap(infos.getUserData()); success = true; @@ -80,6 +83,40 @@ public class DefaultSegmentInfosWriter e } } + /** Save a single segment's info. */ + private void writeInfo(IndexOutput output, SegmentInfo si) throws IOException { + assert si.getDelCount() <= si.docCount: "delCount=" + si.getDelCount() + " docCount=" + si.docCount + " segment=" + si.name; + // Write the Lucene version that created this segment, since 3.1 + output.writeString(si.getVersion()); + output.writeString(si.name); + output.writeInt(si.docCount); + output.writeLong(si.getDelGen()); + + output.writeInt(si.getDocStoreOffset()); + if (si.getDocStoreOffset() != -1) { + output.writeString(si.getDocStoreSegment()); + output.writeByte((byte) (si.getDocStoreIsCompoundFile() ? 1:0)); + } + + Map normGen = si.getNormGen(); + if (normGen == null) { + output.writeInt(SegmentInfo.NO); + } else { + output.writeInt(normGen.size()); + for (Entry entry : normGen.entrySet()) { + output.writeInt(entry.getKey()); + output.writeLong(entry.getValue()); + } + } + + output.writeByte((byte) (si.getUseCompoundFile() ? SegmentInfo.YES : SegmentInfo.NO)); + output.writeInt(si.getDelCount()); + output.writeByte((byte) (si.getHasProxInternal())); + output.writeString(si.getCodec().getName()); + output.writeStringStringMap(si.getDiagnostics()); + output.writeByte((byte) (si.getHasVectorsInternal())); + } + protected IndexOutput createOutput(Directory dir, String segmentFileName, IOContext context) throws IOException { IndexOutput plainOut = dir.createOutput(segmentFileName, context); Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/DefaultStoredFieldsReader.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/DefaultStoredFieldsReader.java?rev=1204297&r1=1204296&r2=1204297&view=diff ============================================================================== --- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/DefaultStoredFieldsReader.java (original) +++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/DefaultStoredFieldsReader.java Sun Nov 20 23:45:25 2011 @@ -32,7 +32,6 @@ import org.apache.lucene.store.AlreadyCl import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; -import org.apache.lucene.util.CloseableThreadLocal; import org.apache.lucene.util.IOUtils; import java.io.Closeable; @@ -50,16 +49,7 @@ public final class DefaultStoredFieldsRe private final static int FORMAT_SIZE = 4; private final FieldInfos fieldInfos; - private CloseableThreadLocal fieldsStreamTL = new CloseableThreadLocal(); - - // The main fieldStream, used only for cloning. - private final IndexInput cloneableFieldsStream; - - // This is a clone of cloneableFieldsStream used for reading documents. - // It should not be cloned outside of a synchronized context. private final IndexInput fieldsStream; - - private final IndexInput cloneableIndexStream; private final IndexInput indexStream; private int numTotalDocs; private int size; @@ -70,8 +60,6 @@ public final class DefaultStoredFieldsRe // file. This will be 0 if we have our own private file. private int docStoreOffset; - private boolean isOriginal = false; - /** Returns a cloned FieldsReader that shares open * IndexInputs with the original one. It is the caller's * job not to close the original FieldsReader until all @@ -80,7 +68,7 @@ public final class DefaultStoredFieldsRe @Override public DefaultStoredFieldsReader clone() { ensureOpen(); - return new DefaultStoredFieldsReader(fieldInfos, numTotalDocs, size, format, docStoreOffset, cloneableFieldsStream, cloneableIndexStream); + return new DefaultStoredFieldsReader(fieldInfos, numTotalDocs, size, format, docStoreOffset, (IndexInput)fieldsStream.clone(), (IndexInput)indexStream.clone()); } /** Verifies that the code version which wrote the segment is supported. */ @@ -102,16 +90,14 @@ public final class DefaultStoredFieldsRe // Used only by clone private DefaultStoredFieldsReader(FieldInfos fieldInfos, int numTotalDocs, int size, int format, int docStoreOffset, - IndexInput cloneableFieldsStream, IndexInput cloneableIndexStream) { + IndexInput fieldsStream, IndexInput indexStream) { this.fieldInfos = fieldInfos; this.numTotalDocs = numTotalDocs; this.size = size; this.format = format; this.docStoreOffset = docStoreOffset; - this.cloneableFieldsStream = cloneableFieldsStream; - this.cloneableIndexStream = cloneableIndexStream; - fieldsStream = (IndexInput) cloneableFieldsStream.clone(); - indexStream = (IndexInput) cloneableIndexStream.clone(); + this.fieldsStream = fieldsStream; + this.indexStream = indexStream; } public DefaultStoredFieldsReader(Directory d, SegmentInfo si, FieldInfos fn, IOContext context) throws IOException { @@ -119,24 +105,20 @@ public final class DefaultStoredFieldsRe final int docStoreOffset = si.getDocStoreOffset(); final int size = si.docCount; boolean success = false; - isOriginal = true; + fieldInfos = fn; try { - fieldInfos = fn; - - cloneableFieldsStream = d.openInput(IndexFileNames.segmentFileName(segment, "", DefaultStoredFieldsWriter.FIELDS_EXTENSION), context); + fieldsStream = d.openInput(IndexFileNames.segmentFileName(segment, "", DefaultStoredFieldsWriter.FIELDS_EXTENSION), context); final String indexStreamFN = IndexFileNames.segmentFileName(segment, "", DefaultStoredFieldsWriter.FIELDS_INDEX_EXTENSION); - cloneableIndexStream = d.openInput(indexStreamFN, context); + indexStream = d.openInput(indexStreamFN, context); - format = cloneableIndexStream.readInt(); + format = indexStream.readInt(); if (format < DefaultStoredFieldsWriter.FORMAT_MINIMUM) - throw new IndexFormatTooOldException(cloneableIndexStream, format, DefaultStoredFieldsWriter.FORMAT_MINIMUM, DefaultStoredFieldsWriter.FORMAT_CURRENT); + throw new IndexFormatTooOldException(indexStream, format, DefaultStoredFieldsWriter.FORMAT_MINIMUM, DefaultStoredFieldsWriter.FORMAT_CURRENT); if (format > DefaultStoredFieldsWriter.FORMAT_CURRENT) - throw new IndexFormatTooNewException(cloneableIndexStream, format, DefaultStoredFieldsWriter.FORMAT_MINIMUM, DefaultStoredFieldsWriter.FORMAT_CURRENT); + throw new IndexFormatTooNewException(indexStream, format, DefaultStoredFieldsWriter.FORMAT_MINIMUM, DefaultStoredFieldsWriter.FORMAT_CURRENT); - fieldsStream = (IndexInput) cloneableFieldsStream.clone(); - - final long indexSize = cloneableIndexStream.length() - FORMAT_SIZE; + final long indexSize = indexStream.length() - FORMAT_SIZE; if (docStoreOffset != -1) { // We read only a slice out of this shared fields file @@ -154,8 +136,6 @@ public final class DefaultStoredFieldsRe throw new CorruptIndexException("doc counts differ for segment " + segment + ": fieldsReader shows " + this.size + " but segmentInfo shows " + si.docCount); } } - - indexStream = (IndexInput) cloneableIndexStream.clone(); numTotalDocs = (int) (indexSize >> 3); success = true; } finally { @@ -180,18 +160,14 @@ public final class DefaultStoredFieldsRe } /** - * Closes the underlying {@link org.apache.lucene.store.IndexInput} streams, including any ones associated with a - * lazy implementation of a Field. This means that the Fields values will not be accessible. + * Closes the underlying {@link org.apache.lucene.store.IndexInput} streams. + * This means that the Fields values will not be accessible. * * @throws IOException */ public final void close() throws IOException { if (!closed) { - if (isOriginal) { - IOUtils.close(fieldsStream, indexStream, fieldsStreamTL, cloneableFieldsStream, cloneableIndexStream); - } else { - IOUtils.close(fieldsStream, indexStream, fieldsStreamTL); - } + IOUtils.close(fieldsStream, indexStream); closed = true; } } Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/DefaultStoredFieldsWriter.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/DefaultStoredFieldsWriter.java?rev=1204297&r1=1204296&r2=1204297&view=diff ============================================================================== --- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/DefaultStoredFieldsWriter.java (original) +++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/DefaultStoredFieldsWriter.java Sun Nov 20 23:45:25 2011 @@ -20,6 +20,7 @@ import java.io.IOException; import org.apache.lucene.document.Document; import org.apache.lucene.index.CorruptIndexException; +import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.IndexFileNames; import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.MergeState; @@ -72,8 +73,8 @@ public final class DefaultStoredFieldsWr /** Extension of stored fields index file */ public static final String FIELDS_INDEX_EXTENSION = "fdx"; - private Directory directory; - private String segment; + private final Directory directory; + private final String segment; private IndexOutput fieldsStream; private IndexOutput indexStream; @@ -118,20 +119,19 @@ public final class DefaultStoredFieldsWr public void abort() { try { close(); - } catch (IOException ignored) { - } + } catch (IOException ignored) {} + try { directory.deleteFile(IndexFileNames.segmentFileName(segment, "", FIELDS_EXTENSION)); - } catch (IOException ignored) { - } + } catch (IOException ignored) {} + try { directory.deleteFile(IndexFileNames.segmentFileName(segment, "", FIELDS_INDEX_EXTENSION)); - } catch (IOException ignored) { - } + } catch (IOException ignored) {} } - public final void writeField(int fieldNumber, IndexableField field) throws IOException { - fieldsStream.writeVInt(fieldNumber); + public final void writeField(FieldInfo info, IndexableField field) throws IOException { + fieldsStream.writeVInt(info.number); int bits = 0; final BytesRef bytes; final String string; Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/FieldsConsumer.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/FieldsConsumer.java?rev=1204297&r1=1204296&r2=1204297&view=diff ============================================================================== --- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/FieldsConsumer.java (original) +++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/FieldsConsumer.java Sun Nov 20 23:45:25 2011 @@ -17,14 +17,14 @@ package org.apache.lucene.index.codecs; * limitations under the License. */ +import java.io.Closeable; +import java.io.IOException; + import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.Fields; import org.apache.lucene.index.FieldsEnum; import org.apache.lucene.index.MergeState; -import org.apache.lucene.index.TermsEnum; - -import java.io.IOException; -import java.io.Closeable; +import org.apache.lucene.index.Terms; /** Abstract API that consumes terms, doc, freq, prox and * payloads postings. Concrete implementations of this @@ -48,10 +48,10 @@ public abstract class FieldsConsumer imp while((field = fieldsEnum.next()) != null) { mergeState.fieldInfo = mergeState.fieldInfos.fieldInfo(field); assert mergeState.fieldInfo != null : "FieldInfo for field is null: "+ field; - TermsEnum terms = fieldsEnum.terms(); + Terms terms = fieldsEnum.terms(); if (terms != null) { final TermsConsumer termsConsumer = addField(mergeState.fieldInfo); - termsConsumer.merge(mergeState, terms); + termsConsumer.merge(mergeState, terms.iterator(null)); } } } Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/StoredFieldsReader.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/StoredFieldsReader.java?rev=1204297&r1=1204296&r2=1204297&view=diff ============================================================================== --- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/StoredFieldsReader.java (original) +++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/StoredFieldsReader.java Sun Nov 20 23:45:25 2011 @@ -1,11 +1,5 @@ package org.apache.lucene.index.codecs; -import java.io.Closeable; -import java.io.IOException; - -import org.apache.lucene.index.CorruptIndexException; -import org.apache.lucene.index.StoredFieldVisitor; - /** * Copyright 2004 The Apache Software Foundation * @@ -22,6 +16,12 @@ import org.apache.lucene.index.StoredFie * the License. */ +import java.io.Closeable; +import java.io.IOException; + +import org.apache.lucene.index.CorruptIndexException; +import org.apache.lucene.index.StoredFieldVisitor; + /** * Codec API for reading stored fields: * Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/StoredFieldsWriter.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/StoredFieldsWriter.java?rev=1204297&r1=1204296&r2=1204297&view=diff ============================================================================== --- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/StoredFieldsWriter.java (original) +++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/StoredFieldsWriter.java Sun Nov 20 23:45:25 2011 @@ -4,6 +4,7 @@ import java.io.Closeable; import java.io.IOException; import org.apache.lucene.document.Document; +import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfos; import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.MergeState; @@ -31,7 +32,7 @@ import org.apache.lucene.util.Bits; *
    *
  1. For every document, {@link #startDocument(int)} is called, * informing the Codec how many fields will be written. - *
  2. {@link #writeField(int, IndexableField)} is called for + *
  3. {@link #writeField(FieldInfo, IndexableField)} is called for * each field in the document. *
  4. After all documents have been written, {@link #finish(int)} * is called for verification/sanity-checks. @@ -43,14 +44,14 @@ import org.apache.lucene.util.Bits; public abstract class StoredFieldsWriter implements Closeable { /** Called before writing the stored fields of the document. - * {@link #writeField(int, IndexableField)} will be called + * {@link #writeField(FieldInfo, IndexableField)} will be called * numStoredFields times. Note that this is * called even if the document has no stored fields, in * this case numStoredFields will be zero. */ public abstract void startDocument(int numStoredFields) throws IOException; /** Writes a single stored field. */ - public abstract void writeField(int fieldNumber, IndexableField field) throws IOException; + public abstract void writeField(FieldInfo info, IndexableField field) throws IOException; /** Aborts writing entirely, implementation should remove * any partially-written files, etc. */ @@ -67,7 +68,7 @@ public abstract class StoredFieldsWriter /** Merges in the stored fields from the readers in * mergeState. The default implementation skips * over deleted documents, and uses {@link #startDocument(int)}, - * {@link #writeField(int, IndexableField)}, and {@link #finish(int)}, + * {@link #writeField(FieldInfo, IndexableField)}, and {@link #finish(int)}, * returning the number of documents that were written. * Implementations can override this method for more sophisticated * merging (bulk-byte copying, etc). */ @@ -110,7 +111,7 @@ public abstract class StoredFieldsWriter for (IndexableField field : doc) { if (field.fieldType().stored()) { - writeField(fieldInfos.fieldNumber(field.name()), field); + writeField(fieldInfos.fieldInfo(field.name()), field); } } } Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/Lucene3xCodec.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/Lucene3xCodec.java?rev=1204297&r1=1204296&r2=1204297&view=diff ============================================================================== --- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/Lucene3xCodec.java (original) +++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/Lucene3xCodec.java Sun Nov 20 23:45:25 2011 @@ -24,14 +24,18 @@ import org.apache.lucene.index.PerDocWri import org.apache.lucene.index.SegmentInfo; import org.apache.lucene.index.SegmentReadState; import org.apache.lucene.index.codecs.Codec; +import org.apache.lucene.index.codecs.DefaultFieldInfosFormat; import org.apache.lucene.index.codecs.DefaultStoredFieldsFormat; import org.apache.lucene.index.codecs.DefaultSegmentInfosFormat; +import org.apache.lucene.index.codecs.DefaultTermVectorsFormat; import org.apache.lucene.index.codecs.DocValuesFormat; +import org.apache.lucene.index.codecs.FieldInfosFormat; import org.apache.lucene.index.codecs.StoredFieldsFormat; import org.apache.lucene.index.codecs.PerDocConsumer; import org.apache.lucene.index.codecs.PerDocValues; import org.apache.lucene.index.codecs.PostingsFormat; import org.apache.lucene.index.codecs.SegmentInfosFormat; +import org.apache.lucene.index.codecs.TermVectorsFormat; import org.apache.lucene.store.Directory; /** @@ -48,6 +52,12 @@ public class Lucene3xCodec extends Codec private final StoredFieldsFormat fieldsFormat = new DefaultStoredFieldsFormat(); // TODO: this should really be a different impl + private final TermVectorsFormat vectorsFormat = new DefaultTermVectorsFormat(); + + // TODO: this should really be a different impl + private final FieldInfosFormat fieldInfosFormat = new DefaultFieldInfosFormat(); + + // TODO: this should really be a different impl // also if we want preflex to *really* be read-only it should throw exception for the writer? // this way IR.commit fails on delete/undelete/setNorm/etc ? private final SegmentInfosFormat infosFormat = new DefaultSegmentInfosFormat(); @@ -82,6 +92,16 @@ public class Lucene3xCodec extends Codec public StoredFieldsFormat storedFieldsFormat() { return fieldsFormat; } + + @Override + public TermVectorsFormat termVectorsFormat() { + return vectorsFormat; + } + + @Override + public FieldInfosFormat fieldInfosFormat() { + return fieldInfosFormat; + } @Override public SegmentInfosFormat segmentInfosFormat() { Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/Lucene3xFields.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/Lucene3xFields.java?rev=1204297&r1=1204296&r2=1204297&view=diff ============================================================================== --- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/Lucene3xFields.java (original) +++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/Lucene3xFields.java Sun Nov 20 23:45:25 2011 @@ -25,7 +25,6 @@ import java.util.Iterator; import java.util.Map; import java.util.TreeMap; -import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.index.DocsAndPositionsEnum; import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.FieldInfo; @@ -38,7 +37,6 @@ import org.apache.lucene.index.Term; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.index.codecs.FieldsProducer; -import org.apache.lucene.store.CompoundFileDirectory; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; @@ -163,6 +161,11 @@ public class Lucene3xFields extends Fiel } @Override + public int getUniqueFieldCount() { + return preTerms.size(); + } + + @Override public long getUniqueTermCount() throws IOException { return getTermsDict().size(); } @@ -196,12 +199,10 @@ public class Lucene3xFields extends Fiel private class PreFlexFieldsEnum extends FieldsEnum { final Iterator it; - private final PreTermsEnum termsEnum; FieldInfo current; public PreFlexFieldsEnum() throws IOException { it = fields.values().iterator(); - termsEnum = new PreTermsEnum(); } @Override @@ -215,9 +216,8 @@ public class Lucene3xFields extends Fiel } @Override - public TermsEnum terms() throws IOException { - termsEnum.reset(current); - return termsEnum; + public Terms terms() throws IOException { + return Lucene3xFields.this.terms(current.name); } } @@ -228,7 +228,7 @@ public class Lucene3xFields extends Fiel } @Override - public TermsEnum iterator() throws IOException { + public TermsEnum iterator(TermsEnum reuse) throws IOException { PreTermsEnum termsEnum = new PreTermsEnum(); termsEnum.reset(fieldInfo); return termsEnum; @@ -994,6 +994,7 @@ public class Lucene3xFields extends Fiel public PreDocsEnum reset(SegmentTermEnum termEnum, Bits liveDocs) throws IOException { docs.setLiveDocs(liveDocs); docs.seek(termEnum); + docID = -1; return this; } @@ -1050,6 +1051,7 @@ public class Lucene3xFields extends Fiel public DocsAndPositionsEnum reset(SegmentTermEnum termEnum, Bits liveDocs) throws IOException { pos.setLiveDocs(liveDocs); pos.seek(termEnum); + docID = -1; return this; } Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40Codec.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40Codec.java?rev=1204297&r1=1204296&r2=1204297&view=diff ============================================================================== --- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40Codec.java (original) +++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40Codec.java Sun Nov 20 23:45:25 2011 @@ -19,12 +19,16 @@ package org.apache.lucene.index.codecs.l import org.apache.lucene.index.codecs.Codec; import org.apache.lucene.index.codecs.DefaultDocValuesFormat; +import org.apache.lucene.index.codecs.DefaultFieldInfosFormat; import org.apache.lucene.index.codecs.DefaultStoredFieldsFormat; import org.apache.lucene.index.codecs.DefaultSegmentInfosFormat; +import org.apache.lucene.index.codecs.DefaultTermVectorsFormat; import org.apache.lucene.index.codecs.DocValuesFormat; +import org.apache.lucene.index.codecs.FieldInfosFormat; import org.apache.lucene.index.codecs.StoredFieldsFormat; import org.apache.lucene.index.codecs.PostingsFormat; import org.apache.lucene.index.codecs.SegmentInfosFormat; +import org.apache.lucene.index.codecs.TermVectorsFormat; import org.apache.lucene.index.codecs.perfield.PerFieldPostingsFormat; /** @@ -39,7 +43,9 @@ import org.apache.lucene.index.codecs.pe // (it writes a minor version, etc). public class Lucene40Codec extends Codec { private final StoredFieldsFormat fieldsFormat = new DefaultStoredFieldsFormat(); + private final TermVectorsFormat vectorsFormat = new DefaultTermVectorsFormat(); private final DocValuesFormat docValuesFormat = new DefaultDocValuesFormat(); + private final FieldInfosFormat fieldInfosFormat = new DefaultFieldInfosFormat(); private final SegmentInfosFormat infosFormat = new DefaultSegmentInfosFormat(); private final PostingsFormat postingsFormat = new PerFieldPostingsFormat() { @Override @@ -56,6 +62,11 @@ public class Lucene40Codec extends Codec public StoredFieldsFormat storedFieldsFormat() { return fieldsFormat; } + + @Override + public TermVectorsFormat termVectorsFormat() { + return vectorsFormat; + } @Override public DocValuesFormat docValuesFormat() { @@ -68,6 +79,11 @@ public class Lucene40Codec extends Codec } @Override + public FieldInfosFormat fieldInfosFormat() { + return fieldInfosFormat; + } + + @Override public SegmentInfosFormat segmentInfosFormat() { return infosFormat; } Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40PostingsReader.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40PostingsReader.java?rev=1204297&r1=1204296&r2=1204297&view=diff ============================================================================== --- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40PostingsReader.java (original) +++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40PostingsReader.java Sun Nov 20 23:45:25 2011 @@ -273,7 +273,8 @@ public class Lucene40PostingsReader exte int limit; // number of docs in this posting int ord; // how many docs we've read - int doc; // doc we last read + int doc = -1; // doc we last read + int accum; // accumulator for doc deltas int freq; // freq we last read Bits liveDocs; @@ -306,7 +307,8 @@ public class Lucene40PostingsReader exte limit = termState.docFreq; assert limit > 0; ord = 0; - doc = 0; + doc = -1; + accum = 0; // if (DEBUG) System.out.println(" sde limit=" + limit + " freqFP=" + freqOffset); skipped = false; @@ -329,9 +331,9 @@ public class Lucene40PostingsReader exte final int code = freqIn.readVInt(); // if (DEBUG) System.out.println(" code=" + code); if (omitTF) { - doc += code; + accum += code; } else { - doc += code >>> 1; // shift off low bit + accum += code >>> 1; // shift off low bit if ((code & 1) != 0) { // if low bit is set freq = 1; // freq is one } else { @@ -339,13 +341,13 @@ public class Lucene40PostingsReader exte } } - if (liveDocs == null || liveDocs.get(doc)) { + if (liveDocs == null || liveDocs.get(accum)) { break; } } //if (DEBUG) System.out.println(" stpr.nextDoc return doc=" + doc); - return doc; + return (doc = accum); } @Override @@ -360,9 +362,9 @@ public class Lucene40PostingsReader exte // manually inlined call to next() for speed final int code = freqIn.readVInt(); if (omitTF) { - doc += code; + accum += code; } else { - doc += code >>> 1; // shift off low bit + accum += code >>> 1; // shift off low bit if ((code & 1) != 0) { // if low bit is set freq = 1; // freq is one } else { @@ -370,8 +372,8 @@ public class Lucene40PostingsReader exte } } - if (liveDocs == null || liveDocs.get(doc)) { - docs[i] = doc; + if (liveDocs == null || liveDocs.get(accum)) { + docs[i] = doc = accum; freqs[i] = freq; ++i; } @@ -422,7 +424,7 @@ public class Lucene40PostingsReader exte // Skipper moved ord = newOrd; - doc = skipper.getDoc(); + doc = accum = skipper.getDoc(); freqIn.seek(skipper.getFreqPointer()); } } @@ -444,7 +446,8 @@ public class Lucene40PostingsReader exte int limit; // number of docs in this posting int ord; // how many docs we've read - int doc; // doc we last read + int doc = -1; // doc we last read + int accum; // accumulator for doc deltas int freq; // freq we last read int position; @@ -482,7 +485,8 @@ public class Lucene40PostingsReader exte assert limit > 0; ord = 0; - doc = 0; + doc = -1; + accum = 0; position = 0; skipped = false; @@ -510,7 +514,7 @@ public class Lucene40PostingsReader exte // Decode next doc/freq pair final int code = freqIn.readVInt(); - doc += code >>> 1; // shift off low bit + accum += code >>> 1; // shift off low bit if ((code & 1) != 0) { // if low bit is set freq = 1; // freq is one } else { @@ -518,7 +522,7 @@ public class Lucene40PostingsReader exte } posPendingCount += freq; - if (liveDocs == null || liveDocs.get(doc)) { + if (liveDocs == null || liveDocs.get(accum)) { break; } } @@ -526,7 +530,7 @@ public class Lucene40PostingsReader exte position = 0; // if (DEBUG) System.out.println(" return doc=" + doc); - return doc; + return (doc = accum); } @Override @@ -572,7 +576,7 @@ public class Lucene40PostingsReader exte if (newOrd > ord) { // Skipper moved ord = newOrd; - doc = skipper.getDoc(); + doc = accum = skipper.getDoc(); freqIn.seek(skipper.getFreqPointer()); lazyProxPointer = skipper.getProxPointer(); posPendingCount = 0; @@ -636,7 +640,8 @@ public class Lucene40PostingsReader exte int limit; // number of docs in this posting int ord; // how many docs we've read - int doc; // doc we last read + int doc = -1; // doc we last read + int accum; // accumulator for doc deltas int freq; // freq we last read int position; @@ -679,7 +684,8 @@ public class Lucene40PostingsReader exte limit = termState.docFreq; ord = 0; - doc = 0; + doc = -1; + accum = 0; position = 0; skipped = false; @@ -707,7 +713,7 @@ public class Lucene40PostingsReader exte // Decode next doc/freq pair final int code = freqIn.readVInt(); - doc += code >>> 1; // shift off low bit + accum += code >>> 1; // shift off low bit if ((code & 1) != 0) { // if low bit is set freq = 1; // freq is one } else { @@ -715,7 +721,7 @@ public class Lucene40PostingsReader exte } posPendingCount += freq; - if (liveDocs == null || liveDocs.get(doc)) { + if (liveDocs == null || liveDocs.get(accum)) { break; } } @@ -723,7 +729,7 @@ public class Lucene40PostingsReader exte position = 0; //System.out.println("StandardR.D&PE nextDoc seg=" + segment + " return doc=" + doc); - return doc; + return (doc = accum); } @Override @@ -769,7 +775,7 @@ public class Lucene40PostingsReader exte if (newOrd > ord) { // Skipper moved ord = newOrd; - doc = skipper.getDoc(); + doc = accum = skipper.getDoc(); freqIn.seek(skipper.getFreqPointer()); lazyProxPointer = skipper.getProxPointer(); posPendingCount = 0; Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40PostingsWriter.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40PostingsWriter.java?rev=1204297&r1=1204296&r2=1204297&view=diff ============================================================================== --- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40PostingsWriter.java (original) +++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40PostingsWriter.java Sun Nov 20 23:45:25 2011 @@ -26,8 +26,8 @@ import java.util.List; import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.index.DocsEnum; -import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfo.IndexOptions; +import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.IndexFileNames; import org.apache.lucene.index.SegmentWriteState; import org.apache.lucene.index.codecs.PostingsWriterBase; @@ -36,6 +36,7 @@ import org.apache.lucene.store.IndexOutp import org.apache.lucene.store.RAMOutputStream; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.CodecUtil; +import org.apache.lucene.util.IOUtils; /** @lucene.experimental */ public final class Lucene40PostingsWriter extends PostingsWriterBase { @@ -92,14 +93,22 @@ public final class Lucene40PostingsWrite // this.segment = state.segmentName; String fileName = IndexFileNames.segmentFileName(state.segmentName, state.segmentSuffix, Lucene40PostingsFormat.FREQ_EXTENSION); freqOut = state.directory.createOutput(fileName, state.context); - if (state.fieldInfos.hasProx()) { - // At least one field does not omit TF, so create the - // prox file - fileName = IndexFileNames.segmentFileName(state.segmentName, state.segmentSuffix, Lucene40PostingsFormat.PROX_EXTENSION); - proxOut = state.directory.createOutput(fileName, state.context); - } else { - // Every field omits TF so we will write no prox file - proxOut = null; + boolean success = false; + try { + if (state.fieldInfos.hasProx()) { + // At least one field does not omit TF, so create the + // prox file + fileName = IndexFileNames.segmentFileName(state.segmentName, state.segmentSuffix, Lucene40PostingsFormat.PROX_EXTENSION); + proxOut = state.directory.createOutput(fileName, state.context); + } else { + // Every field omits TF so we will write no prox file + proxOut = null; + } + success = true; + } finally { + if (!success) { + IOUtils.closeWhileHandlingException(freqOut); + } } totalNumDocs = state.numDocs; Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/memory/MemoryPostingsFormat.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/memory/MemoryPostingsFormat.java?rev=1204297&r1=1204296&r2=1204297&view=diff ============================================================================== --- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/memory/MemoryPostingsFormat.java (original) +++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/memory/MemoryPostingsFormat.java Sun Nov 20 23:45:25 2011 @@ -273,7 +273,8 @@ public class MemoryPostingsFormat extend private Bits liveDocs; private int docUpto; - private int docID; + private int docID = -1; + private int accum; private int freq; private int payloadLen; private int numDocs; @@ -295,7 +296,8 @@ public class MemoryPostingsFormat extend in.reset(buffer, 0, bufferIn.length - bufferIn.offset); System.arraycopy(bufferIn.bytes, bufferIn.offset, buffer, 0, bufferIn.length - bufferIn.offset); this.liveDocs = liveDocs; - docID = 0; + docID = -1; + accum = 0; docUpto = 0; payloadLen = 0; this.numDocs = numDocs; @@ -314,12 +316,12 @@ public class MemoryPostingsFormat extend } docUpto++; if (indexOptions == IndexOptions.DOCS_ONLY) { - docID += in.readVInt(); + accum += in.readVInt(); freq = 1; } else { final int code = in.readVInt(); - docID += code >>> 1; - if (VERBOSE) System.out.println(" docID=" + docID + " code=" + code); + accum += code >>> 1; + if (VERBOSE) System.out.println(" docID=" + accum + " code=" + code); if ((code & 1) != 0) { freq = 1; } else { @@ -343,9 +345,9 @@ public class MemoryPostingsFormat extend } } - if (liveDocs == null || liveDocs.get(docID)) { - if (VERBOSE) System.out.println(" return docID=" + docID + " freq=" + freq); - return docID; + if (liveDocs == null || liveDocs.get(accum)) { + if (VERBOSE) System.out.println(" return docID=" + accum + " freq=" + freq); + return (docID = accum); } } } @@ -380,7 +382,8 @@ public class MemoryPostingsFormat extend private Bits liveDocs; private int docUpto; - private int docID; + private int docID = -1; + private int accum; private int freq; private int numDocs; private int posPending; @@ -412,7 +415,8 @@ public class MemoryPostingsFormat extend in.reset(buffer, 0, bufferIn.length - bufferIn.offset); System.arraycopy(bufferIn.bytes, bufferIn.offset, buffer, 0, bufferIn.length - bufferIn.offset); this.liveDocs = liveDocs; - docID = 0; + docID = -1; + accum = 0; docUpto = 0; payload.bytes = buffer; payloadLength = 0; @@ -436,7 +440,7 @@ public class MemoryPostingsFormat extend docUpto++; final int code = in.readVInt(); - docID += code >>> 1; + accum += code >>> 1; if ((code & 1) != 0) { freq = 1; } else { @@ -444,11 +448,11 @@ public class MemoryPostingsFormat extend assert freq > 0; } - if (liveDocs == null || liveDocs.get(docID)) { + if (liveDocs == null || liveDocs.get(accum)) { pos = 0; posPending = freq; - if (VERBOSE) System.out.println(" return docID=" + docID + " freq=" + freq); - return docID; + if (VERBOSE) System.out.println(" return docID=" + accum + " freq=" + freq); + return (docID = accum); } // Skip positions @@ -723,7 +727,7 @@ public class MemoryPostingsFormat extend } @Override - public TermsEnum iterator() { + public TermsEnum iterator(TermsEnum reuse) { return new FSTTermsEnum(field, fst); } @@ -768,8 +772,9 @@ public class MemoryPostingsFormat extend return current.field.name; } - public TermsEnum terms() { - return current.iterator(); + @Override + public Terms terms() { + return current; } }; } @@ -780,6 +785,11 @@ public class MemoryPostingsFormat extend } @Override + public int getUniqueFieldCount() { + return fields.size(); + } + + @Override public void close() { // Drop ref to FST: for(TermsReader termsReader : fields.values()) { Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/perfield/PerFieldPostingsFormat.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/perfield/PerFieldPostingsFormat.java?rev=1204297&r1=1204296&r2=1204297&view=diff ============================================================================== --- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/perfield/PerFieldPostingsFormat.java (original) +++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/perfield/PerFieldPostingsFormat.java Sun Nov 20 23:45:25 2011 @@ -34,7 +34,6 @@ import org.apache.lucene.index.SegmentIn import org.apache.lucene.index.SegmentReadState; import org.apache.lucene.index.SegmentWriteState; import org.apache.lucene.index.Terms; -import org.apache.lucene.index.TermsEnum; import org.apache.lucene.index.codecs.FieldsConsumer; import org.apache.lucene.index.codecs.FieldsProducer; import org.apache.lucene.index.codecs.PostingsFormat; @@ -213,7 +212,7 @@ public abstract class PerFieldPostingsFo } @Override - public String next() { + public String next() throws IOException { if (it.hasNext()) { current = it.next(); } else { @@ -224,13 +223,8 @@ public abstract class PerFieldPostingsFo } @Override - public TermsEnum terms() throws IOException { - final Terms terms = fields.get(current).terms(current); - if (terms != null) { - return terms.iterator(); - } else { - return TermsEnum.EMPTY; - } + public Terms terms() throws IOException { + return fields.get(current).terms(current); } } @@ -246,6 +240,11 @@ public abstract class PerFieldPostingsFo } @Override + public int getUniqueFieldCount() { + return fields.size(); + } + + @Override public void close() throws IOException { IOUtils.close(formats.values()); } Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingPostingsReader.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingPostingsReader.java?rev=1204297&r1=1204296&r2=1204297&view=diff ============================================================================== --- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingPostingsReader.java (original) +++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingPostingsReader.java Sun Nov 20 23:45:25 2011 @@ -257,7 +257,8 @@ public class PulsingPostingsReader exten private final IndexOptions indexOptions; private final boolean storePayloads; private Bits liveDocs; - private int docID; + private int docID = -1; + private int accum; private int freq; private int payloadLength; @@ -279,7 +280,8 @@ public class PulsingPostingsReader exten } System.arraycopy(termState.postings, 0, postingsBytes, 0, termState.postingsSize); postings.reset(postingsBytes, 0, termState.postingsSize); - docID = 0; + docID = -1; + accum = 0; payloadLength = 0; freq = 1; this.liveDocs = liveDocs; @@ -302,9 +304,9 @@ public class PulsingPostingsReader exten final int code = postings.readVInt(); //System.out.println(" read code=" + code); if (indexOptions == IndexOptions.DOCS_ONLY) { - docID += code; + accum += code; } else { - docID += code >>> 1; // shift off low bit + accum += code >>> 1; // shift off low bit if ((code & 1) != 0) { // if low bit is set freq = 1; // freq is one } else { @@ -332,8 +334,8 @@ public class PulsingPostingsReader exten } } - if (liveDocs == null || liveDocs.get(docID)) { - return docID; + if (liveDocs == null || liveDocs.get(accum)) { + return (docID = accum); } } } @@ -365,7 +367,8 @@ public class PulsingPostingsReader exten private final boolean storePayloads; private Bits liveDocs; - private int docID; + private int docID = -1; + private int accum; private int freq; private int posPending; private int position; @@ -394,7 +397,8 @@ public class PulsingPostingsReader exten this.liveDocs = liveDocs; payloadLength = 0; posPending = 0; - docID = 0; + docID = -1; + accum = 0; //System.out.println("PR d&p reset storesPayloads=" + storePayloads + " bytes=" + bytes.length + " this=" + this); return this; } @@ -414,7 +418,7 @@ public class PulsingPostingsReader exten } final int code = postings.readVInt(); - docID += code >>> 1; // shift off low bit + accum += code >>> 1; // shift off low bit if ((code & 1) != 0) { // if low bit is set freq = 1; // freq is one } else { @@ -422,10 +426,10 @@ public class PulsingPostingsReader exten } posPending = freq; - if (liveDocs == null || liveDocs.get(docID)) { + if (liveDocs == null || liveDocs.get(accum)) { //System.out.println(" return docID=" + docID + " freq=" + freq); position = 0; - return docID; + return (docID = accum); } } } Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/sep/SepPostingsReader.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/sep/SepPostingsReader.java?rev=1204297&r1=1204296&r2=1204297&view=diff ============================================================================== --- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/sep/SepPostingsReader.java (original) +++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/sep/SepPostingsReader.java Sun Nov 20 23:45:25 2011 @@ -312,7 +312,8 @@ public class SepPostingsReader extends P class SepDocsEnum extends DocsEnum { int docFreq; - int doc; + int doc = -1; + int accum; int count; int freq; long freqStart; @@ -376,7 +377,8 @@ public class SepPostingsReader extends P // NOTE: unused if docFreq < skipMinimum: skipFP = termState.skipFP; count = 0; - doc = 0; + doc = -1; + accum = 0; skipped = false; return this; @@ -394,18 +396,18 @@ public class SepPostingsReader extends P // Decode next doc //System.out.println("decode docDelta:"); - doc += docReader.next(); + accum += docReader.next(); if (!omitTF) { //System.out.println("decode freq:"); freq = freqReader.next(); } - if (liveDocs == null || liveDocs.get(doc)) { + if (liveDocs == null || liveDocs.get(accum)) { break; } } - return doc; + return (doc = accum); } @Override @@ -420,14 +422,14 @@ public class SepPostingsReader extends P count++; // manually inlined call to next() for speed //System.out.println("decode doc"); - doc += docReader.next(); + accum += docReader.next(); if (!omitTF) { //System.out.println("decode freq"); freq = freqReader.next(); } - if (liveDocs == null || liveDocs.get(doc)) { - docs[i] = doc; + if (liveDocs == null || liveDocs.get(accum)) { + docs[i] = doc = accum; freqs[i] = freq; //System.out.println(" docs[" + i + "]=" + doc + " count=" + count + " dF=" + docFreq); i++; @@ -488,7 +490,7 @@ public class SepPostingsReader extends P } skipper.getDocIndex().seek(docReader); count = newCount; - doc = skipper.getDoc(); + doc = accum = skipper.getDoc(); } } @@ -505,7 +507,8 @@ public class SepPostingsReader extends P class SepDocsAndPositionsEnum extends DocsAndPositionsEnum { int docFreq; - int doc; + int doc = -1; + int accum; int count; int freq; long freqStart; @@ -572,7 +575,8 @@ public class SepPostingsReader extends P docFreq = termState.docFreq; count = 0; - doc = 0; + doc = -1; + accum = 0; pendingPosCount = 0; pendingPayloadBytes = 0; skipped = false; @@ -595,20 +599,20 @@ public class SepPostingsReader extends P // Decode next doc //System.out.println(" sep d&p read doc"); - doc += docReader.next(); + accum += docReader.next(); //System.out.println(" sep d&p read freq"); freq = freqReader.next(); pendingPosCount += freq; - if (liveDocs == null || liveDocs.get(doc)) { + if (liveDocs == null || liveDocs.get(accum)) { break; } } position = 0; - return doc; + return (doc = accum); } @Override @@ -668,7 +672,7 @@ public class SepPostingsReader extends P posIndex.set(skipper.getPosIndex()); posSeekPending = true; count = newCount; - doc = skipper.getDoc(); + doc = accum = skipper.getDoc(); //System.out.println(" moved to doc=" + doc); //payloadIn.seek(skipper.getPayloadPointer()); payloadFP = skipper.getPayloadPointer(); Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/sep/SepPostingsWriter.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/sep/SepPostingsWriter.java?rev=1204297&r1=1204296&r2=1204297&view=diff ============================================================================== --- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/sep/SepPostingsWriter.java (original) +++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/sep/SepPostingsWriter.java Sun Nov 20 23:45:25 2011 @@ -151,7 +151,6 @@ public final class SepPostingsWriter ext if (!success) { IOUtils.closeWhileHandlingException(docOut, skipOut, freqOut, posOut, payloadOut); } - } } Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextCodec.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextCodec.java?rev=1204297&r1=1204296&r2=1204297&view=diff ============================================================================== --- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextCodec.java (original) +++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextCodec.java Sun Nov 20 23:45:25 2011 @@ -19,11 +19,12 @@ package org.apache.lucene.index.codecs.s import org.apache.lucene.index.codecs.Codec; import org.apache.lucene.index.codecs.DefaultDocValuesFormat; -import org.apache.lucene.index.codecs.DefaultSegmentInfosFormat; import org.apache.lucene.index.codecs.DocValuesFormat; +import org.apache.lucene.index.codecs.FieldInfosFormat; import org.apache.lucene.index.codecs.PostingsFormat; import org.apache.lucene.index.codecs.SegmentInfosFormat; import org.apache.lucene.index.codecs.StoredFieldsFormat; +import org.apache.lucene.index.codecs.TermVectorsFormat; /** * plain text index format. @@ -34,11 +35,11 @@ import org.apache.lucene.index.codecs.St public final class SimpleTextCodec extends Codec { private final PostingsFormat postings = new SimpleTextPostingsFormat(); private final StoredFieldsFormat storedFields = new SimpleTextStoredFieldsFormat(); - + private final SegmentInfosFormat segmentInfos = new SimpleTextSegmentInfosFormat(); + private final FieldInfosFormat fieldInfosFormat = new SimpleTextFieldInfosFormat(); + private final TermVectorsFormat vectorsFormat = new SimpleTextTermVectorsFormat(); // TODO: need a plain-text impl private final DocValuesFormat docValues = new DefaultDocValuesFormat(); - // TODO: need a plain-text impl - private final SegmentInfosFormat segmentInfos = new DefaultSegmentInfosFormat(); public SimpleTextCodec() { super("SimpleText"); @@ -58,6 +59,16 @@ public final class SimpleTextCodec exten public StoredFieldsFormat storedFieldsFormat() { return storedFields; } + + @Override + public TermVectorsFormat termVectorsFormat() { + return vectorsFormat; + } + + @Override + public FieldInfosFormat fieldInfosFormat() { + return fieldInfosFormat; + } @Override public SegmentInfosFormat segmentInfosFormat() { Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsReader.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsReader.java?rev=1204297&r1=1204296&r2=1204297&view=diff ============================================================================== --- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsReader.java (original) +++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsReader.java Sun Nov 20 23:45:25 2011 @@ -87,8 +87,8 @@ class SimpleTextFieldsReader extends Fie } @Override - public TermsEnum terms() throws IOException { - return SimpleTextFieldsReader.this.terms(current).iterator(); + public Terms terms() throws IOException { + return SimpleTextFieldsReader.this.terms(current); } } @@ -314,7 +314,7 @@ class SimpleTextFieldsReader extends Fie private class SimpleTextDocsAndPositionsEnum extends DocsAndPositionsEnum { private final IndexInput inStart; private final IndexInput in; - private int docID; + private int docID = -1; private int tf; private Bits liveDocs; private final BytesRef scratch = new BytesRef(10); @@ -336,6 +336,7 @@ class SimpleTextFieldsReader extends Fie public SimpleTextDocsAndPositionsEnum reset(long fp, Bits liveDocs) { this.liveDocs = liveDocs; nextDocStart = fp; + docID = -1; return this; } @@ -523,7 +524,7 @@ class SimpleTextFieldsReader extends Fie } @Override - public TermsEnum iterator() throws IOException { + public TermsEnum iterator(TermsEnum reuse) throws IOException { if (fst != null) { return new SimpleTextTermsEnum(fst, indexOptions); } else { @@ -582,6 +583,11 @@ class SimpleTextFieldsReader extends Fie } @Override + public int getUniqueFieldCount() { + return -1; + } + + @Override public void close() throws IOException { in.close(); } Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextStoredFieldsWriter.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextStoredFieldsWriter.java?rev=1204297&r1=1204296&r2=1204297&view=diff ============================================================================== --- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextStoredFieldsWriter.java (original) +++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextStoredFieldsWriter.java Sun Nov 20 23:45:25 2011 @@ -19,6 +19,7 @@ package org.apache.lucene.index.codecs.s import java.io.IOException; +import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.IndexFileNames; import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.codecs.StoredFieldsWriter; @@ -87,9 +88,9 @@ public class SimpleTextStoredFieldsWrite } @Override - public void writeField(int fieldNumber, IndexableField field) throws IOException { + public void writeField(FieldInfo info, IndexableField field) throws IOException { write(FIELD); - write(Integer.toString(fieldNumber)); + write(Integer.toString(info.number)); newLine(); write(NAME); Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/values/MultiIndexDocValues.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/values/MultiIndexDocValues.java?rev=1204297&r1=1204296&r2=1204297&view=diff ============================================================================== --- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/values/MultiIndexDocValues.java (original) +++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/values/MultiIndexDocValues.java Sun Nov 20 23:45:25 2011 @@ -169,6 +169,7 @@ public class MultiIndexDocValues extends } } + // TODO: this is dup of IndexDocValues.getDefaultSource()? private static class EmptySource extends Source { public EmptySource(ValueType type) { Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/FieldCacheImpl.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/FieldCacheImpl.java?rev=1204297&r1=1204296&r2=1204297&view=diff ============================================================================== --- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/FieldCacheImpl.java (original) +++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/FieldCacheImpl.java Sun Nov 20 23:45:25 2011 @@ -333,7 +333,7 @@ class FieldCacheImpl implements FieldCac setDocsWithField = false; } } - final TermsEnum termsEnum = terms.iterator(); + final TermsEnum termsEnum = terms.iterator(null); DocsEnum docs = null; try { while(true) { @@ -406,7 +406,7 @@ class FieldCacheImpl implements FieldCac setDocsWithField = false; } } - final TermsEnum termsEnum = terms.iterator(); + final TermsEnum termsEnum = terms.iterator(null); DocsEnum docs = null; try { while(true) { @@ -505,7 +505,7 @@ class FieldCacheImpl implements FieldCac setDocsWithField = false; } } - final TermsEnum termsEnum = terms.iterator(); + final TermsEnum termsEnum = terms.iterator(null); DocsEnum docs = null; try { while(true) { @@ -574,7 +574,7 @@ class FieldCacheImpl implements FieldCac // Fast case: all docs have this field: return new Bits.MatchAllBits(maxDoc); } - final TermsEnum termsEnum = terms.iterator(); + final TermsEnum termsEnum = terms.iterator(null); DocsEnum docs = null; while(true) { final BytesRef term = termsEnum.next(); @@ -655,7 +655,7 @@ class FieldCacheImpl implements FieldCac setDocsWithField = false; } } - final TermsEnum termsEnum = terms.iterator(); + final TermsEnum termsEnum = terms.iterator(null); DocsEnum docs = null; try { while(true) { @@ -743,7 +743,7 @@ class FieldCacheImpl implements FieldCac setDocsWithField = false; } } - final TermsEnum termsEnum = terms.iterator(); + final TermsEnum termsEnum = terms.iterator(null); DocsEnum docs = null; try { while(true) { @@ -832,7 +832,7 @@ class FieldCacheImpl implements FieldCac setDocsWithField = false; } } - final TermsEnum termsEnum = terms.iterator(); + final TermsEnum termsEnum = terms.iterator(null); DocsEnum docs = null; try { while(true) { @@ -1128,7 +1128,7 @@ class FieldCacheImpl implements FieldCac int termOrd = 1; if (terms != null) { - final TermsEnum termsEnum = terms.iterator(); + final TermsEnum termsEnum = terms.iterator(null); DocsEnum docs = null; while(true) { @@ -1253,7 +1253,7 @@ class FieldCacheImpl implements FieldCac if (terms != null) { int termCount = 0; - final TermsEnum termsEnum = terms.iterator(); + final TermsEnum termsEnum = terms.iterator(null); DocsEnum docs = null; while(true) { if (termCount++ == termCountHardLimit) { Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/FieldComparator.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/FieldComparator.java?rev=1204297&r1=1204296&r2=1204297&view=diff ============================================================================== --- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/FieldComparator.java (original) +++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/FieldComparator.java Sun Nov 20 23:45:25 2011 @@ -1670,11 +1670,17 @@ public abstract class FieldComparator final IndexDocValues dv = context.reader.docValues(field); if (dv == null) { + // This may mean entire segment had no docs with + // this DV field; use default field value (empty + // byte[]) in this case: termsIndex = IndexDocValues.getDefaultSortedSource(ValueType.BYTES_VAR_SORTED, context.reader.maxDoc()); } else { termsIndex = dv.getSource().asSortedSource(); if (termsIndex == null) { - termsIndex = IndexDocValues.getDefaultSortedSource(ValueType.BYTES_VAR_SORTED, context.reader.maxDoc()); + // This means segment has doc values, but they are + // not able to provide a sorted source; consider + // this a hard error: + throw new IllegalStateException("DocValues exist for field \"" + field + "\", but not as a sorted source: type=" + dv.getSource().type() + " reader=" + context.reader); } } Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/FuzzyQuery.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/FuzzyQuery.java?rev=1204297&r1=1204296&r2=1204297&view=diff ============================================================================== --- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/FuzzyQuery.java (original) +++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/FuzzyQuery.java Sun Nov 20 23:45:25 2011 @@ -139,7 +139,7 @@ public class FuzzyQuery extends MultiTer @Override protected TermsEnum getTermsEnum(Terms terms, AttributeSource atts) throws IOException { if (!termLongEnough) { // can only match if it's exact - return new SingleTermsEnum(terms.iterator(), term.bytes()); + return new SingleTermsEnum(terms.iterator(null), term.bytes()); } return new FuzzyTermsEnum(terms, atts, getTerm(), minimumSimilarity, prefixLength); } Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/FuzzyTermsEnum.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/FuzzyTermsEnum.java?rev=1204297&r1=1204296&r2=1204297&view=diff ============================================================================== --- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/FuzzyTermsEnum.java (original) +++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/FuzzyTermsEnum.java Sun Nov 20 23:45:25 2011 @@ -406,7 +406,7 @@ public final class FuzzyTermsEnum extend * @throws IOException */ public LinearFuzzyTermsEnum() throws IOException { - super(terms.iterator()); + super(terms.iterator(null)); this.text = new int[termLength - realPrefixLength]; System.arraycopy(termText, realPrefixLength, text, 0, text.length); Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/IndexSearcher.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/IndexSearcher.java?rev=1204297&r1=1204296&r2=1204297&view=diff ============================================================================== --- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/IndexSearcher.java (original) +++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/IndexSearcher.java Sun Nov 20 23:45:25 2011 @@ -41,7 +41,6 @@ import org.apache.lucene.index.Term; import org.apache.lucene.index.Terms; import org.apache.lucene.search.similarities.DefaultSimilarityProvider; import org.apache.lucene.search.similarities.SimilarityProvider; -import org.apache.lucene.store.Directory; import org.apache.lucene.store.NIOFSDirectory; // javadoc import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; @@ -76,7 +75,6 @@ import org.apache.lucene.util.ThreadInte */ public class IndexSearcher implements Closeable { final IndexReader reader; // package private for testing! - private boolean closeReader; // NOTE: these members might change in incompatible ways // in the next release @@ -105,34 +103,9 @@ public class IndexSearcher implements Cl /** The SimilarityProvider implementation used by this searcher. */ private SimilarityProvider similarityProvider = defaultProvider; - /** Creates a searcher searching the index in the named - * directory, with readOnly=true - * @param path directory where IndexReader will be opened - * @throws CorruptIndexException if the index is corrupt - * @throws IOException if there is a low-level IO error - */ - public IndexSearcher(Directory path) throws CorruptIndexException, IOException { - this(IndexReader.open(path, true), true, null); - } - - /** Creates a searcher searching the index in the named - * directory. You should pass readOnly=true, since it - * gives much better concurrent performance, unless you - * intend to do write operations (delete documents or - * change norms) with the underlying IndexReader. - * @param path directory where IndexReader will be opened - * @param readOnly if true, the underlying IndexReader - * will be opened readOnly - * @throws CorruptIndexException if the index is corrupt - * @throws IOException if there is a low-level IO error - */ - public IndexSearcher(Directory path, boolean readOnly) throws CorruptIndexException, IOException { - this(IndexReader.open(path, readOnly), true, null); - } - /** Creates a searcher searching the provided index. */ public IndexSearcher(IndexReader r) { - this(r, false, null); + this(r, null); } /** Runs searches for each segment separately, using the @@ -147,7 +120,7 @@ public class IndexSearcher implements Cl * * @lucene.experimental */ public IndexSearcher(IndexReader r, ExecutorService executor) { - this(r, false, executor); + this(r.getTopReaderContext(), executor); } /** @@ -167,7 +140,12 @@ public class IndexSearcher implements Cl * @lucene.experimental */ public IndexSearcher(ReaderContext context, ExecutorService executor) { - this(context, false, executor); + assert context.isTopLevel: "IndexSearcher's ReaderContext must be topLevel for reader" + context.reader; + reader = context.reader; + this.executor = executor; + this.readerContext = context; + leafContexts = ReaderUtil.leaves(context); + this.leafSlices = executor == null ? null : slices(leafContexts); } /** @@ -178,22 +156,7 @@ public class IndexSearcher implements Cl * @lucene.experimental */ public IndexSearcher(ReaderContext context) { - this(context, (ExecutorService) null); - } - - // convenience ctor for other IR based ctors - private IndexSearcher(IndexReader reader, boolean closeReader, ExecutorService executor) { - this(reader.getTopReaderContext(), closeReader, executor); - } - - private IndexSearcher(ReaderContext context, boolean closeReader, ExecutorService executor) { - assert context.isTopLevel: "IndexSearcher's ReaderContext must be topLevel for reader" + context.reader; - reader = context.reader; - this.executor = executor; - this.closeReader = closeReader; - this.readerContext = context; - leafContexts = ReaderUtil.leaves(context); - this.leafSlices = executor == null ? null : slices(leafContexts); + this(context, null); } /** @@ -236,17 +199,8 @@ public class IndexSearcher implements Cl return similarityProvider; } - /** - * Note that the underlying IndexReader is not closed, if - * IndexSearcher was constructed with IndexSearcher(IndexReader r). - * If the IndexReader was supplied implicitly by specifying a directory, then - * the IndexReader is closed. - */ @Override public void close() throws IOException { - if (closeReader) { - reader.close(); - } } /** @lucene.internal */