From commits-return-101364-archive-asf-public=cust-asf.ponee.io@lucene.apache.org Mon Jun 4 08:40:43 2018 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx-eu-01.ponee.io (Postfix) with SMTP id 151CE180674 for ; Mon, 4 Jun 2018 08:40:41 +0200 (CEST) Received: (qmail 44351 invoked by uid 500); 4 Jun 2018 06:40:41 -0000 Mailing-List: contact commits-help@lucene.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@lucene.apache.org Delivered-To: mailing list commits@lucene.apache.org Received: (qmail 44341 invoked by uid 99); 4 Jun 2018 06:40:41 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 04 Jun 2018 06:40:41 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id B20E9DFB1C; Mon, 4 Jun 2018 06:40:40 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: simonw@apache.org To: commits@lucene.apache.org Date: Mon, 04 Jun 2018 06:40:41 -0000 Message-Id: <28bd86bd55094b56b935de0ea3a795ca@git.apache.org> In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [2/2] lucene-solr:branch_7x: LUCENE-8335: Enforce soft-deletes field up-front LUCENE-8335: Enforce soft-deletes field up-front Soft deletes field must be marked as such once it's introduced and can't be changed after the fact. Co-authored-by: Nhat Nguyen Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/67b6593e Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/67b6593e Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/67b6593e Branch: refs/heads/branch_7x Commit: 67b6593e7adbce76532e285cef42118e6cc3448f Parents: 6f6b7f5 Author: Simon Willnauer Authored: Mon Jun 4 08:23:31 2018 +0200 Committer: Simon Willnauer Committed: Mon Jun 4 08:40:20 2018 +0200 ---------------------------------------------------------------------- lucene/CHANGES.txt | 4 + .../simpletext/SimpleTextFieldInfosFormat.java | 11 +- .../lucene50/Lucene50FieldInfosFormat.java | 2 +- .../lucene60/Lucene60FieldInfosFormat.java | 8 +- .../java/org/apache/lucene/index/FieldInfo.java | 18 +++- .../org/apache/lucene/index/FieldInfos.java | 51 ++++++--- .../org/apache/lucene/index/IndexWriter.java | 8 +- .../org/apache/lucene/index/MultiFields.java | 6 +- .../apache/lucene/index/ParallelLeafReader.java | 7 +- .../org/apache/lucene/index/TestCodecs.java | 4 +- .../test/org/apache/lucene/index/TestDoc.java | 2 +- .../apache/lucene/index/TestFieldsReader.java | 2 +- .../apache/lucene/index/TestIndexWriter.java | 106 +++++++++++++++++++ .../lucene/index/TestPendingSoftDeletes.java | 10 +- .../apache/lucene/index/TestSegmentMerger.java | 2 +- .../search/highlight/TermVectorLeafReader.java | 2 +- .../apache/lucene/index/memory/MemoryIndex.java | 4 +- .../index/BaseFieldInfoFormatTestCase.java | 14 +-- .../index/BaseIndexFileFormatTestCase.java | 2 +- .../lucene/index/MismatchedLeafReader.java | 3 +- .../lucene/index/RandomPostingsTester.java | 4 +- .../solr/handler/component/ExpandComponent.java | 3 +- .../solr/search/CollapsingQParserPlugin.java | 2 +- .../java/org/apache/solr/search/Insanity.java | 2 +- .../solr/uninverting/UninvertingReader.java | 2 +- 25 files changed, 224 insertions(+), 55 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/67b6593e/lucene/CHANGES.txt ---------------------------------------------------------------------- diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index 54c874d..44829fa 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -99,6 +99,10 @@ New Features now use to also take pending deletes into account which ensures that all file generations per segment always go forward. (Simon Willnauer) +* LUCENE-8335: Enforce soft-deletes field up-front. Soft deletes field must be marked + as such once it's introduced and can't be changed after the fact. + (Nhat Nguyen via Simon Willnauer) + Bug Fixes * LUCENE-8221: MoreLikeThis.setMaxDocFreqPct can easily int-overflow on larger http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/67b6593e/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosFormat.java ---------------------------------------------------------------------- diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosFormat.java b/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosFormat.java index 0ace153..1c40cbd 100644 --- a/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosFormat.java +++ b/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosFormat.java @@ -66,6 +66,7 @@ public class SimpleTextFieldInfosFormat extends FieldInfosFormat { static final BytesRef ATT_VALUE = new BytesRef(" value "); static final BytesRef DIM_COUNT = new BytesRef(" dimensional count "); static final BytesRef DIM_NUM_BYTES = new BytesRef(" dimensional num bytes "); + static final BytesRef SOFT_DELETES = new BytesRef(" soft-deletes "); @Override public FieldInfos read(Directory directory, SegmentInfo segmentInfo, String segmentSuffix, IOContext iocontext) throws IOException { @@ -140,9 +141,13 @@ public class SimpleTextFieldInfosFormat extends FieldInfosFormat { assert StringHelper.startsWith(scratch.get(), DIM_NUM_BYTES); int dimensionalNumBytes = Integer.parseInt(readString(DIM_NUM_BYTES.length, scratch)); + SimpleTextUtil.readLine(input, scratch); + assert StringHelper.startsWith(scratch.get(), SOFT_DELETES); + boolean isSoftDeletesField = Boolean.parseBoolean(readString(SOFT_DELETES.length, scratch)); + infos[i] = new FieldInfo(name, fieldNumber, storeTermVector, omitNorms, storePayloads, indexOptions, docValuesType, dvGen, Collections.unmodifiableMap(atts), - dimensionalCount, dimensionalNumBytes); + dimensionalCount, dimensionalNumBytes, isSoftDeletesField); } SimpleTextUtil.checkFooter(input); @@ -238,6 +243,10 @@ public class SimpleTextFieldInfosFormat extends FieldInfosFormat { SimpleTextUtil.write(out, DIM_NUM_BYTES); SimpleTextUtil.write(out, Integer.toString(fi.getPointNumBytes()), scratch); SimpleTextUtil.writeNewline(out); + + SimpleTextUtil.write(out, SOFT_DELETES); + SimpleTextUtil.write(out, Boolean.toString(fi.isSoftDeletesField()), scratch); + SimpleTextUtil.writeNewline(out); } SimpleTextUtil.writeChecksum(out, scratch); success = true; http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/67b6593e/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50FieldInfosFormat.java ---------------------------------------------------------------------- diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50FieldInfosFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50FieldInfosFormat.java index a76bfeb..30dca70 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50FieldInfosFormat.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50FieldInfosFormat.java @@ -148,7 +148,7 @@ public final class Lucene50FieldInfosFormat extends FieldInfosFormat { lastAttributes = attributes; try { infos[i] = new FieldInfo(name, fieldNumber, storeTermVector, omitNorms, storePayloads, - indexOptions, docValuesType, dvGen, attributes, 0, 0); + indexOptions, docValuesType, dvGen, attributes, 0, 0, false); infos[i].checkConsistency(); } catch (IllegalStateException e) { throw new CorruptIndexException("invalid fieldinfo for field: " + name + ", fieldNumber=" + fieldNumber, input, e); http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/67b6593e/lucene/core/src/java/org/apache/lucene/codecs/lucene60/Lucene60FieldInfosFormat.java ---------------------------------------------------------------------- diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene60/Lucene60FieldInfosFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene60/Lucene60FieldInfosFormat.java index a35461e..522a73f 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene60/Lucene60FieldInfosFormat.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene60/Lucene60FieldInfosFormat.java @@ -136,6 +136,7 @@ public final class Lucene60FieldInfosFormat extends FieldInfosFormat { boolean storeTermVector = (bits & STORE_TERMVECTOR) != 0; boolean omitNorms = (bits & OMIT_NORMS) != 0; boolean storePayloads = (bits & STORE_PAYLOADS) != 0; + boolean isSoftDeletesField = (bits & SOFT_DELETES_FIELD) != 0; final IndexOptions indexOptions = getIndexOptions(input, input.readByte()); @@ -159,7 +160,7 @@ public final class Lucene60FieldInfosFormat extends FieldInfosFormat { try { infos[i] = new FieldInfo(name, fieldNumber, storeTermVector, omitNorms, storePayloads, indexOptions, docValuesType, dvGen, attributes, - pointDimensionCount, pointNumBytes); + pointDimensionCount, pointNumBytes, isSoftDeletesField); infos[i].checkConsistency(); } catch (IllegalStateException e) { throw new CorruptIndexException("invalid fieldinfo for field: " + name + ", fieldNumber=" + fieldNumber, input, e); @@ -277,6 +278,7 @@ public final class Lucene60FieldInfosFormat extends FieldInfosFormat { if (fi.hasVectors()) bits |= STORE_TERMVECTOR; if (fi.omitsNorms()) bits |= OMIT_NORMS; if (fi.hasPayloads()) bits |= STORE_PAYLOADS; + if (fi.isSoftDeletesField()) bits |= SOFT_DELETES_FIELD; output.writeByte(bits); output.writeByte(indexOptionsByte(fi.getIndexOptions())); @@ -301,10 +303,12 @@ public final class Lucene60FieldInfosFormat extends FieldInfosFormat { // Codec header static final String CODEC_NAME = "Lucene60FieldInfos"; static final int FORMAT_START = 0; - static final int FORMAT_CURRENT = FORMAT_START; + static final int FORMAT_SOFT_DELETES = 1; + static final int FORMAT_CURRENT = FORMAT_SOFT_DELETES; // Field flags static final byte STORE_TERMVECTOR = 0x1; static final byte OMIT_NORMS = 0x2; static final byte STORE_PAYLOADS = 0x4; + static final byte SOFT_DELETES_FIELD = 0x8; } http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/67b6593e/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java ---------------------------------------------------------------------- diff --git a/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java b/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java index 422292b..250c94b 100644 --- a/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java +++ b/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java @@ -53,14 +53,17 @@ public final class FieldInfo { private int pointDimensionCount; private int pointNumBytes; + // whether this field is used as the soft-deletes field + private final boolean softDeletesField; + /** * Sole constructor. * * @lucene.experimental */ - public FieldInfo(String name, int number, boolean storeTermVector, boolean omitNorms, - boolean storePayloads, IndexOptions indexOptions, DocValuesType docValues, - long dvGen, Map attributes, int pointDimensionCount, int pointNumBytes) { + public FieldInfo(String name, int number, boolean storeTermVector, boolean omitNorms, boolean storePayloads, + IndexOptions indexOptions, DocValuesType docValues, long dvGen, Map attributes, + int pointDimensionCount, int pointNumBytes, boolean softDeletesField) { this.name = Objects.requireNonNull(name); this.number = number; this.docValuesType = Objects.requireNonNull(docValues, "DocValuesType must not be null (field: \"" + name + "\")"); @@ -78,6 +81,7 @@ public final class FieldInfo { this.attributes = Objects.requireNonNull(attributes); this.pointDimensionCount = pointDimensionCount; this.pointNumBytes = pointNumBytes; + this.softDeletesField = softDeletesField; assert checkConsistency(); } @@ -334,4 +338,12 @@ public final class FieldInfo { public Map attributes() { return attributes; } + + /** + * Returns true if this field is configured and used as the soft-deletes field. + * See {@link IndexWriterConfig#softDeletesField} + */ + public boolean isSoftDeletesField() { + return softDeletesField; + } } http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/67b6593e/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java ---------------------------------------------------------------------- diff --git a/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java b/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java index f47e15c..22ca9de7 100644 --- a/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java +++ b/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java @@ -43,6 +43,7 @@ public class FieldInfos implements Iterable { private final boolean hasNorms; private final boolean hasDocValues; private final boolean hasPointValues; + private final String softDeletesField; // used only by fieldInfo(int) private final FieldInfo[] byNumber; @@ -62,6 +63,7 @@ public class FieldInfos implements Iterable { boolean hasNorms = false; boolean hasDocValues = false; boolean hasPointValues = false; + String softDeletesField = null; int size = 0; // number of elements in byNumberTemp, number of used array slots FieldInfo[] byNumberTemp = new FieldInfo[10]; // initial array capacity of 10 @@ -92,6 +94,12 @@ public class FieldInfos implements Iterable { hasDocValues |= info.getDocValuesType() != DocValuesType.NONE; hasPayloads |= info.hasPayloads(); hasPointValues |= (info.getPointDimensionCount() != 0); + if (info.isSoftDeletesField()) { + if (softDeletesField != null && softDeletesField.equals(info.name) == false) { + throw new IllegalArgumentException("multiple soft-deletes fields [" + info.name + ", " + softDeletesField + "]"); + } + softDeletesField = info.name; + } } this.hasVectors = hasVectors; @@ -102,6 +110,7 @@ public class FieldInfos implements Iterable { this.hasNorms = hasNorms; this.hasDocValues = hasDocValues; this.hasPointValues = hasPointValues; + this.softDeletesField = softDeletesField; List valuesTemp = new ArrayList<>(); byNumber = new FieldInfo[size]; @@ -153,6 +162,11 @@ public class FieldInfos implements Iterable { public boolean hasPointValues() { return hasPointValues; } + + /** Returns the soft-deletes field name if exists; otherwise returns null */ + public String getSoftDeletesField() { + return softDeletesField; + } /** Returns the number of fields */ public int size() { @@ -220,12 +234,16 @@ public class FieldInfos implements Iterable { // norms back on after they were already ommitted; today // we silently discard the norm but this is badly trappy private int lowestUnassignedFieldNumber = -1; + + // The soft-deletes field from IWC to enforce a single soft-deletes field + private final String softDeletesFieldName; - FieldNumbers() { + FieldNumbers(String softDeletesFieldName) { this.nameToNumber = new HashMap<>(); this.numberToName = new HashMap<>(); this.docValuesType = new HashMap<>(); this.dimensions = new HashMap<>(); + this.softDeletesFieldName = softDeletesFieldName; } /** @@ -234,7 +252,7 @@ public class FieldInfos implements Iterable { * number assigned if possible otherwise the first unassigned field number * is used as the field number. */ - synchronized int addOrGet(String fieldName, int preferredFieldNumber, DocValuesType dvType, int dimensionCount, int dimensionNumBytes) { + synchronized int addOrGet(String fieldName, int preferredFieldNumber, DocValuesType dvType, int dimensionCount, int dimensionNumBytes, boolean isSoftDeletesField) { if (dvType != DocValuesType.NONE) { DocValuesType currentDVType = docValuesType.get(fieldName); if (currentDVType == null) { @@ -274,6 +292,16 @@ public class FieldInfos implements Iterable { nameToNumber.put(fieldName, fieldNumber); } + if (isSoftDeletesField) { + if (softDeletesFieldName == null) { + throw new IllegalArgumentException("this index has [" + fieldName + "] as soft-deletes already but soft-deletes field is not configured in IWC"); + } else if (fieldName.equals(softDeletesFieldName) == false) { + throw new IllegalArgumentException("cannot configure [" + softDeletesFieldName + "] as soft-deletes; this index uses [" + fieldName + "] as soft-deletes already"); + } + } else if (fieldName.equals(softDeletesFieldName)) { + throw new IllegalArgumentException("cannot configure [" + softDeletesFieldName + "] as soft-deletes; this index uses [" + fieldName + "] as non-soft-deletes already"); + } + return fieldNumber.intValue(); } @@ -354,11 +382,7 @@ public class FieldInfos implements Iterable { private final HashMap byName = new HashMap<>(); final FieldNumbers globalFieldNumbers; private boolean finished; - - Builder() { - this(new FieldNumbers()); - } - + /** * Creates a new instance with the given {@link FieldNumbers}. */ @@ -384,8 +408,9 @@ public class FieldInfos implements Iterable { // number for this field. If the field was seen // before then we'll get the same name and number, // else we'll allocate a new one: - final int fieldNumber = globalFieldNumbers.addOrGet(name, -1, DocValuesType.NONE, 0, 0); - fi = new FieldInfo(name, fieldNumber, false, false, false, IndexOptions.NONE, DocValuesType.NONE, -1, new HashMap<>(), 0, 0); + final boolean isSoftDeletesField = name.equals(globalFieldNumbers.softDeletesFieldName); + final int fieldNumber = globalFieldNumbers.addOrGet(name, -1, DocValuesType.NONE, 0, 0, isSoftDeletesField); + fi = new FieldInfo(name, fieldNumber, false, false, false, IndexOptions.NONE, DocValuesType.NONE, -1, new HashMap<>(), 0, 0, isSoftDeletesField); assert !byName.containsKey(fi.name); globalFieldNumbers.verifyConsistent(Integer.valueOf(fi.number), fi.name, DocValuesType.NONE); byName.put(fi.name, fi); @@ -398,7 +423,7 @@ public class FieldInfos implements Iterable { boolean storeTermVector, boolean omitNorms, boolean storePayloads, IndexOptions indexOptions, DocValuesType docValues, long dvGen, - int dimensionCount, int dimensionNumBytes) { + int dimensionCount, int dimensionNumBytes, boolean isSoftDeletesField) { assert assertNotFinished(); if (docValues == null) { throw new NullPointerException("DocValuesType must not be null"); @@ -410,8 +435,8 @@ public class FieldInfos implements Iterable { // number for this field. If the field was seen // before then we'll get the same name and number, // else we'll allocate a new one: - final int fieldNumber = globalFieldNumbers.addOrGet(name, preferredFieldNumber, docValues, dimensionCount, dimensionNumBytes); - fi = new FieldInfo(name, fieldNumber, storeTermVector, omitNorms, storePayloads, indexOptions, docValues, dvGen, new HashMap<>(), dimensionCount, dimensionNumBytes); + final int fieldNumber = globalFieldNumbers.addOrGet(name, preferredFieldNumber, docValues, dimensionCount, dimensionNumBytes, isSoftDeletesField); + fi = new FieldInfo(name, fieldNumber, storeTermVector, omitNorms, storePayloads, indexOptions, docValues, dvGen, new HashMap<>(), dimensionCount, dimensionNumBytes, isSoftDeletesField); assert !byName.containsKey(fi.name); globalFieldNumbers.verifyConsistent(Integer.valueOf(fi.number), fi.name, fi.getDocValuesType()); byName.put(fi.name, fi); @@ -444,7 +469,7 @@ public class FieldInfos implements Iterable { return addOrUpdateInternal(fi.name, fi.number, fi.hasVectors(), fi.omitsNorms(), fi.hasPayloads(), fi.getIndexOptions(), fi.getDocValuesType(), dvGen, - fi.getPointDimensionCount(), fi.getPointNumBytes()); + fi.getPointDimensionCount(), fi.getPointNumBytes(), fi.isSoftDeletesField()); } public FieldInfo fieldInfo(String fieldName) { http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/67b6593e/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java ---------------------------------------------------------------------- diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java index ba30df0..144d2c0 100644 --- a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java +++ b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java @@ -962,12 +962,12 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable, * If this {@link SegmentInfos} has no global field number map the returned instance is empty */ private FieldNumbers getFieldNumberMap() throws IOException { - final FieldNumbers map = new FieldNumbers(); + final FieldNumbers map = new FieldNumbers(config.softDeletesField); for(SegmentCommitInfo info : segmentInfos) { FieldInfos fis = readFieldInfos(info); for(FieldInfo fi : fis) { - map.addOrGet(fi.name, fi.number, fi.getDocValuesType(), fi.getPointDimensionCount(), fi.getPointNumBytes()); + map.addOrGet(fi.name, fi.number, fi.getDocValuesType(), fi.getPointDimensionCount(), fi.getPointNumBytes(), fi.isSoftDeletesField()); } } @@ -1789,7 +1789,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable, if (globalFieldNumberMap.contains(f.name(), dvType) == false) { // if this field doesn't exists we try to add it. if it exists and the DV type doesn't match we // get a consistent error message as if you try to do that during an indexing operation. - globalFieldNumberMap.addOrGet(f.name(), -1, dvType, 0, 0); + globalFieldNumberMap.addOrGet(f.name(), -1, dvType, 0, 0, f.name().equals(config.softDeletesField)); assert globalFieldNumberMap.contains(f.name(), dvType); } if (config.getIndexSortFields().contains(f.name())) { @@ -2826,7 +2826,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable, FieldInfos fis = readFieldInfos(info); for(FieldInfo fi : fis) { // This will throw exceptions if any of the incoming fields have an illegal schema change: - globalFieldNumberMap.addOrGet(fi.name, fi.number, fi.getDocValuesType(), fi.getPointDimensionCount(), fi.getPointNumBytes()); + globalFieldNumberMap.addOrGet(fi.name, fi.number, fi.getDocValuesType(), fi.getPointDimensionCount(), fi.getPointNumBytes(), fi.isSoftDeletesField()); } infos.add(copySegmentAsIs(info, newSegName, context)); } http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/67b6593e/lucene/core/src/java/org/apache/lucene/index/MultiFields.java ---------------------------------------------------------------------- diff --git a/lucene/core/src/java/org/apache/lucene/index/MultiFields.java b/lucene/core/src/java/org/apache/lucene/index/MultiFields.java index 1a7b15b..19078a8 100644 --- a/lucene/core/src/java/org/apache/lucene/index/MultiFields.java +++ b/lucene/core/src/java/org/apache/lucene/index/MultiFields.java @@ -25,6 +25,7 @@ import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.concurrent.ConcurrentHashMap; import org.apache.lucene.util.Bits; @@ -263,7 +264,10 @@ public final class MultiFields extends Fields { * will be unavailable. */ public static FieldInfos getMergedFieldInfos(IndexReader reader) { - final FieldInfos.Builder builder = new FieldInfos.Builder(); + final String softDeletesField = reader.leaves().stream() + .map(l -> l.reader().getFieldInfos().getSoftDeletesField()) + .filter(Objects::nonNull).findAny().orElse(null); + final FieldInfos.Builder builder = new FieldInfos.Builder(new FieldInfos.FieldNumbers(softDeletesField)); for(final LeafReaderContext ctx : reader.leaves()) { builder.add(ctx.reader().getFieldInfos()); } http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/67b6593e/lucene/core/src/java/org/apache/lucene/index/ParallelLeafReader.java ---------------------------------------------------------------------- diff --git a/lucene/core/src/java/org/apache/lucene/index/ParallelLeafReader.java b/lucene/core/src/java/org/apache/lucene/index/ParallelLeafReader.java index 492b6e7..25f200a 100644 --- a/lucene/core/src/java/org/apache/lucene/index/ParallelLeafReader.java +++ b/lucene/core/src/java/org/apache/lucene/index/ParallelLeafReader.java @@ -23,6 +23,7 @@ import java.util.HashMap; import java.util.IdentityHashMap; import java.util.Iterator; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.SortedMap; import java.util.TreeMap; @@ -101,9 +102,11 @@ public class ParallelLeafReader extends LeafReader { throw new IllegalArgumentException("All readers must have same maxDoc: "+maxDoc+"!="+reader.maxDoc()); } } - + final String softDeletesField = completeReaderSet.stream() + .map(r -> r.getFieldInfos().getSoftDeletesField()) + .filter(Objects::nonNull).findAny().orElse(null); // TODO: make this read-only in a cleaner way? - FieldInfos.Builder builder = new FieldInfos.Builder(); + FieldInfos.Builder builder = new FieldInfos.Builder(new FieldInfos.FieldNumbers(softDeletesField)); Sort indexSort = null; int createdVersionMajor = -1; http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/67b6593e/lucene/core/src/test/org/apache/lucene/index/TestCodecs.java ---------------------------------------------------------------------- diff --git a/lucene/core/src/test/org/apache/lucene/index/TestCodecs.java b/lucene/core/src/test/org/apache/lucene/index/TestCodecs.java index 4625f73..9eb2541 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestCodecs.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestCodecs.java @@ -210,7 +210,7 @@ public class TestCodecs extends LuceneTestCase { terms[i] = new TermData(text, docs, null); } - final FieldInfos.Builder builder = new FieldInfos.Builder(); + final FieldInfos.Builder builder = new FieldInfos.Builder(new FieldInfos.FieldNumbers(null)); final FieldData field = new FieldData("field", builder, terms, true, false); final FieldData[] fields = new FieldData[] {field}; @@ -257,7 +257,7 @@ public class TestCodecs extends LuceneTestCase { } public void testRandomPostings() throws Throwable { - final FieldInfos.Builder builder = new FieldInfos.Builder(); + final FieldInfos.Builder builder = new FieldInfos.Builder(new FieldInfos.FieldNumbers(null)); final FieldData[] fields = new FieldData[NUM_FIELDS]; for(int i=0;iasList(r1, r2), si, InfoStream.getDefault(), trackingDir, - new FieldInfos.FieldNumbers(), context); + new FieldInfos.FieldNumbers(null), context); MergeState mergeState = merger.merge(); r1.close(); http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/67b6593e/lucene/core/src/test/org/apache/lucene/index/TestFieldsReader.java ---------------------------------------------------------------------- diff --git a/lucene/core/src/test/org/apache/lucene/index/TestFieldsReader.java b/lucene/core/src/test/org/apache/lucene/index/TestFieldsReader.java index 48d69ec..ce24b7f 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestFieldsReader.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestFieldsReader.java @@ -44,7 +44,7 @@ public class TestFieldsReader extends LuceneTestCase { @BeforeClass public static void beforeClass() throws Exception { testDoc = new Document(); - fieldInfos = new FieldInfos.Builder(); + fieldInfos = new FieldInfos.Builder(new FieldInfos.FieldNumbers(null)); DocHelper.setupDoc(testDoc); for (IndexableField field : testDoc.getFields()) { FieldInfo fieldInfo = fieldInfos.getOrAdd(field.name()); http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/67b6593e/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java ---------------------------------------------------------------------- diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java index 6db7e86..98da021 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java @@ -3399,4 +3399,110 @@ public class TestIndexWriter extends LuceneTestCase { IOUtils.close(reader, writer, dir); } + public void testPreventChangingSoftDeletesField() throws Exception { + Directory dir = newDirectory(); + IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig().setSoftDeletesField("my_deletes")); + Document v1 = new Document(); + v1.add(new StringField("id", "1", Field.Store.YES)); + v1.add(new StringField("version", "1", Field.Store.YES)); + writer.addDocument(v1); + Document v2 = new Document(); + v2.add(new StringField("id", "1", Field.Store.YES)); + v2.add(new StringField("version", "2", Field.Store.YES)); + writer.softUpdateDocument(new Term("id", "1"), v2, new NumericDocValuesField("my_deletes", 1)); + writer.commit(); + writer.close(); + for (SegmentCommitInfo si : SegmentInfos.readLatestCommit(dir)) { + FieldInfos fieldInfos = IndexWriter.readFieldInfos(si); + assertEquals("my_deletes", fieldInfos.getSoftDeletesField()); + assertTrue(fieldInfos.fieldInfo("my_deletes").isSoftDeletesField()); + } + + IllegalArgumentException illegalError = expectThrows(IllegalArgumentException.class, () -> { + new IndexWriter(dir, newIndexWriterConfig().setSoftDeletesField("your_deletes")); + }); + assertEquals("cannot configure [your_deletes] as soft-deletes; " + + "this index uses [my_deletes] as soft-deletes already", illegalError.getMessage()); + + IndexWriterConfig softDeleteConfig = newIndexWriterConfig().setSoftDeletesField("my_deletes") + .setMergePolicy(new SoftDeletesRetentionMergePolicy("my_deletes", () -> new MatchAllDocsQuery(), newMergePolicy())); + writer = new IndexWriter(dir, softDeleteConfig); + Document tombstone = new Document(); + tombstone.add(new StringField("id", "tombstone", Field.Store.YES)); + tombstone.add(new NumericDocValuesField("my_deletes", 1)); + writer.addDocument(tombstone); + writer.flush(); + for (SegmentCommitInfo si : writer.segmentInfos) { + FieldInfos fieldInfos = IndexWriter.readFieldInfos(si); + assertEquals("my_deletes", fieldInfos.getSoftDeletesField()); + assertTrue(fieldInfos.fieldInfo("my_deletes").isSoftDeletesField()); + } + writer.close(); + // reopen writer without soft-deletes field should be prevented + IllegalArgumentException reopenError = expectThrows(IllegalArgumentException.class, () -> { + new IndexWriter(dir, newIndexWriterConfig()); + }); + assertEquals("this index has [my_deletes] as soft-deletes already" + + " but soft-deletes field is not configured in IWC", reopenError.getMessage()); + dir.close(); + } + + public void testPreventAddingIndexesWithDifferentSoftDeletesField() throws Exception { + Directory dir1 = newDirectory(); + IndexWriter w1 = new IndexWriter(dir1, newIndexWriterConfig().setSoftDeletesField("soft_deletes_1")); + for (int i = 0; i < 2; i++) { + Document d = new Document(); + d.add(new StringField("id", "1", Field.Store.YES)); + d.add(new StringField("version", Integer.toString(i), Field.Store.YES)); + w1.softUpdateDocument(new Term("id", "1"), d, new NumericDocValuesField("soft_deletes_1", 1)); + } + w1.commit(); + w1.close(); + + Directory dir2 = newDirectory(); + IndexWriter w2 = new IndexWriter(dir2, newIndexWriterConfig().setSoftDeletesField("soft_deletes_2")); + IllegalArgumentException error = expectThrows(IllegalArgumentException.class, () -> w2.addIndexes(dir1)); + assertEquals("cannot configure [soft_deletes_2] as soft-deletes; this index uses [soft_deletes_1] as soft-deletes already", + error.getMessage()); + w2.close(); + + Directory dir3 = newDirectory(); + IndexWriterConfig config = newIndexWriterConfig().setSoftDeletesField("soft_deletes_1"); + IndexWriter w3 = new IndexWriter(dir3, config); + w3.addIndexes(dir1); + for (SegmentCommitInfo si : w3.segmentInfos) { + FieldInfo softDeleteField = IndexWriter.readFieldInfos(si).fieldInfo("soft_deletes_1"); + assertTrue(softDeleteField.isSoftDeletesField()); + } + w3.close(); + IOUtils.close(dir1, dir2, dir3); + } + + public void testNotAllowUsingExistingFieldAsSoftDeletes() throws Exception { + Directory dir = newDirectory(); + IndexWriter w = new IndexWriter(dir, newIndexWriterConfig()); + for (int i = 0; i < 2; i++) { + Document d = new Document(); + d.add(new StringField("id", "1", Field.Store.YES)); + if (random().nextBoolean()) { + d.add(new NumericDocValuesField("dv_field", 1)); + w.updateDocument(new Term("id", "1"), d); + } else { + w.softUpdateDocument(new Term("id", "1"), d, new NumericDocValuesField("dv_field", 1)); + } + } + w.commit(); + w.close(); + String softDeletesField = random().nextBoolean() ? "id" : "dv_field"; + IllegalArgumentException error = expectThrows(IllegalArgumentException.class, () -> { + IndexWriterConfig config = newIndexWriterConfig().setSoftDeletesField(softDeletesField); + new IndexWriter(dir, config); + }); + assertEquals("cannot configure [" + softDeletesField + "] as soft-deletes;" + + " this index uses [" + softDeletesField + "] as non-soft-deletes already", error.getMessage()); + IndexWriterConfig config = newIndexWriterConfig().setSoftDeletesField("non-existing-field"); + w = new IndexWriter(dir, config); + w.close(); + dir.close(); + } } http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/67b6593e/lucene/core/src/test/org/apache/lucene/index/TestPendingSoftDeletes.java ---------------------------------------------------------------------- diff --git a/lucene/core/src/test/org/apache/lucene/index/TestPendingSoftDeletes.java b/lucene/core/src/test/org/apache/lucene/index/TestPendingSoftDeletes.java index 5fadd3f..3047364 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestPendingSoftDeletes.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestPendingSoftDeletes.java @@ -120,7 +120,7 @@ public class TestPendingSoftDeletes extends TestPendingDeletes { deletes.onNewReader(segmentReader, commitInfo); reader.close(); writer.close(); - FieldInfo fieldInfo = new FieldInfo("_soft_deletes", 1, false, false, false, IndexOptions.NONE, DocValuesType.NUMERIC, 0, Collections.emptyMap(), 0, 0); + FieldInfo fieldInfo = new FieldInfo("_soft_deletes", 1, false, false, false, IndexOptions.NONE, DocValuesType.NUMERIC, 0, Collections.emptyMap(), 0, 0, true); List docsDeleted = Arrays.asList(1, 3, 7, 8, DocIdSetIterator.NO_MORE_DOCS); List updates = Arrays.asList(singleUpdate(docsDeleted, 10, true)); for (DocValuesFieldUpdates update : updates) { @@ -140,7 +140,7 @@ public class TestPendingSoftDeletes extends TestPendingDeletes { docsDeleted = Arrays.asList(1, 2, DocIdSetIterator.NO_MORE_DOCS); updates = Arrays.asList(singleUpdate(docsDeleted, 10, true)); - fieldInfo = new FieldInfo("_soft_deletes", 1, false, false, false, IndexOptions.NONE, DocValuesType.NUMERIC, 1, Collections.emptyMap(), 0, 0); + fieldInfo = new FieldInfo("_soft_deletes", 1, false, false, false, IndexOptions.NONE, DocValuesType.NUMERIC, 1, Collections.emptyMap(), 0, 0, true); for (DocValuesFieldUpdates update : updates) { deletes.onDocValuesUpdate(fieldInfo, update.iterator()); } @@ -182,7 +182,7 @@ public class TestPendingSoftDeletes extends TestPendingDeletes { SegmentCommitInfo segmentInfo = segmentReader.getSegmentInfo(); PendingDeletes deletes = newPendingDeletes(segmentInfo); deletes.onNewReader(segmentReader, segmentInfo); - FieldInfo fieldInfo = new FieldInfo("_soft_deletes", 1, false, false, false, IndexOptions.NONE, DocValuesType.NUMERIC, segmentInfo.getNextDocValuesGen(), Collections.emptyMap(), 0, 0); + FieldInfo fieldInfo = new FieldInfo("_soft_deletes", 1, false, false, false, IndexOptions.NONE, DocValuesType.NUMERIC, segmentInfo.getNextDocValuesGen(), Collections.emptyMap(), 0, 0, true); List docsDeleted = Arrays.asList(1, DocIdSetIterator.NO_MORE_DOCS); List updates = Arrays.asList(singleUpdate(docsDeleted, 3, true)); for (DocValuesFieldUpdates update : updates) { @@ -228,7 +228,7 @@ public class TestPendingSoftDeletes extends TestPendingDeletes { SegmentCommitInfo segmentInfo = segmentReader.getSegmentInfo(); PendingDeletes deletes = newPendingDeletes(segmentInfo); deletes.onNewReader(segmentReader, segmentInfo); - FieldInfo fieldInfo = new FieldInfo("_soft_deletes", 1, false, false, false, IndexOptions.NONE, DocValuesType.NUMERIC, segmentInfo.getNextDocValuesGen(), Collections.emptyMap(), 0, 0); + FieldInfo fieldInfo = new FieldInfo("_soft_deletes", 1, false, false, false, IndexOptions.NONE, DocValuesType.NUMERIC, segmentInfo.getNextDocValuesGen(), Collections.emptyMap(), 0, 0, true); List updates = Arrays.asList(singleUpdate(Arrays.asList(0, 1, DocIdSetIterator.NO_MORE_DOCS), 3, false)); for (DocValuesFieldUpdates update : updates) { deletes.onDocValuesUpdate(fieldInfo, update.iterator()); @@ -247,7 +247,7 @@ public class TestPendingSoftDeletes extends TestPendingDeletes { assertEquals(0, deletes.numPendingDeletes()); segmentInfo.advanceDocValuesGen(); - fieldInfo = new FieldInfo("_soft_deletes", 1, false, false, false, IndexOptions.NONE, DocValuesType.NUMERIC, segmentInfo.getNextDocValuesGen(), Collections.emptyMap(), 0, 0); + fieldInfo = new FieldInfo("_soft_deletes", 1, false, false, false, IndexOptions.NONE, DocValuesType.NUMERIC, segmentInfo.getNextDocValuesGen(), Collections.emptyMap(), 0, 0, true); updates = Arrays.asList(singleUpdate(Arrays.asList(1, DocIdSetIterator.NO_MORE_DOCS), 3, true)); for (DocValuesFieldUpdates update : updates) { deletes.onDocValuesUpdate(fieldInfo, update.iterator()); http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/67b6593e/lucene/core/src/test/org/apache/lucene/index/TestSegmentMerger.java ---------------------------------------------------------------------- diff --git a/lucene/core/src/test/org/apache/lucene/index/TestSegmentMerger.java b/lucene/core/src/test/org/apache/lucene/index/TestSegmentMerger.java index 6d0e04b..1171b90 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestSegmentMerger.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestSegmentMerger.java @@ -88,7 +88,7 @@ public class TestSegmentMerger extends LuceneTestCase { SegmentMerger merger = new SegmentMerger(Arrays.asList(reader1, reader2), si, InfoStream.getDefault(), mergedDir, - new FieldInfos.FieldNumbers(), + new FieldInfos.FieldNumbers(null), newIOContext(random(), new IOContext(new MergeInfo(-1, -1, false, -1)))); MergeState mergeState = merger.merge(); int docsMerged = mergeState.segmentInfo.maxDoc(); http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/67b6593e/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TermVectorLeafReader.java ---------------------------------------------------------------------- diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TermVectorLeafReader.java b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TermVectorLeafReader.java index 144209d..1eef95f 100644 --- a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TermVectorLeafReader.java +++ b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TermVectorLeafReader.java @@ -81,7 +81,7 @@ public class TermVectorLeafReader extends LeafReader { } FieldInfo fieldInfo = new FieldInfo(field, 0, true, true, terms.hasPayloads(), - indexOptions, DocValuesType.NONE, -1, Collections.emptyMap(), 0, 0); + indexOptions, DocValuesType.NONE, -1, Collections.emptyMap(), 0, 0, false); fieldInfos = new FieldInfos(new FieldInfo[]{fieldInfo}); } http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/67b6593e/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java ---------------------------------------------------------------------- diff --git a/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java b/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java index f732cf3..48e2bd2 100644 --- a/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java +++ b/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java @@ -500,7 +500,7 @@ public class MemoryIndex { IndexOptions indexOptions = storeOffsets ? IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS : IndexOptions.DOCS_AND_FREQS_AND_POSITIONS; return new FieldInfo(fieldName, ord, fieldType.storeTermVectors(), fieldType.omitNorms(), storePayloads, indexOptions, fieldType.docValuesType(), -1, Collections.emptyMap(), - fieldType.pointDimensionCount(), fieldType.pointNumBytes()); + fieldType.pointDimensionCount(), fieldType.pointNumBytes(), false); } private void storePointValues(Info info, BytesRef pointValue) { @@ -519,7 +519,7 @@ public class MemoryIndex { info.fieldInfo = new FieldInfo( info.fieldInfo.name, info.fieldInfo.number, info.fieldInfo.hasVectors(), info.fieldInfo.hasPayloads(), info.fieldInfo.hasPayloads(), info.fieldInfo.getIndexOptions(), docValuesType, -1, info.fieldInfo.attributes(), - info.fieldInfo.getPointDimensionCount(), info.fieldInfo.getPointNumBytes() + info.fieldInfo.getPointDimensionCount(), info.fieldInfo.getPointNumBytes(), info.fieldInfo.isSoftDeletesField() ); } else if (existingDocValuesType != docValuesType) { throw new IllegalArgumentException("Can't add [" + docValuesType + "] doc values field [" + fieldName + "], because [" + existingDocValuesType + "] doc values field already exists"); http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/67b6593e/lucene/test-framework/src/java/org/apache/lucene/index/BaseFieldInfoFormatTestCase.java ---------------------------------------------------------------------- diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/BaseFieldInfoFormatTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/index/BaseFieldInfoFormatTestCase.java index 9363ce6..3515b9a 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/index/BaseFieldInfoFormatTestCase.java +++ b/lucene/test-framework/src/java/org/apache/lucene/index/BaseFieldInfoFormatTestCase.java @@ -53,7 +53,7 @@ public abstract class BaseFieldInfoFormatTestCase extends BaseIndexFileFormatTes Directory dir = newDirectory(); Codec codec = getCodec(); SegmentInfo segmentInfo = newSegmentInfo(dir, "_123"); - FieldInfos.Builder builder = new FieldInfos.Builder(); + FieldInfos.Builder builder = new FieldInfos.Builder(new FieldInfos.FieldNumbers(null)); FieldInfo fi = builder.getOrAdd("field"); fi.setIndexOptions(TextField.TYPE_STORED.indexOptions()); addAttributes(fi); @@ -75,7 +75,7 @@ public abstract class BaseFieldInfoFormatTestCase extends BaseIndexFileFormatTes Directory dir = newDirectory(); Codec codec = getCodec(); SegmentInfo segmentInfo = newSegmentInfo(dir, "_123"); - FieldInfos.Builder builder = new FieldInfos.Builder(); + FieldInfos.Builder builder = new FieldInfos.Builder(new FieldInfos.FieldNumbers(null)); FieldInfo fi = builder.getOrAdd("field"); fi.setIndexOptions(TextField.TYPE_STORED.indexOptions()); addAttributes(fi); @@ -115,7 +115,7 @@ public abstract class BaseFieldInfoFormatTestCase extends BaseIndexFileFormatTes dir.failOn(fail); Codec codec = getCodec(); SegmentInfo segmentInfo = newSegmentInfo(dir, "_123"); - FieldInfos.Builder builder = new FieldInfos.Builder(); + FieldInfos.Builder builder = new FieldInfos.Builder(new FieldInfos.FieldNumbers(null)); FieldInfo fi = builder.getOrAdd("field"); fi.setIndexOptions(TextField.TYPE_STORED.indexOptions()); addAttributes(fi); @@ -150,7 +150,7 @@ public abstract class BaseFieldInfoFormatTestCase extends BaseIndexFileFormatTes dir.failOn(fail); Codec codec = getCodec(); SegmentInfo segmentInfo = newSegmentInfo(dir, "_123"); - FieldInfos.Builder builder = new FieldInfos.Builder(); + FieldInfos.Builder builder = new FieldInfos.Builder(new FieldInfos.FieldNumbers(null)); FieldInfo fi = builder.getOrAdd("field"); fi.setIndexOptions(TextField.TYPE_STORED.indexOptions()); addAttributes(fi); @@ -185,7 +185,7 @@ public abstract class BaseFieldInfoFormatTestCase extends BaseIndexFileFormatTes dir.failOn(fail); Codec codec = getCodec(); SegmentInfo segmentInfo = newSegmentInfo(dir, "_123"); - FieldInfos.Builder builder = new FieldInfos.Builder(); + FieldInfos.Builder builder = new FieldInfos.Builder(new FieldInfos.FieldNumbers(null)); FieldInfo fi = builder.getOrAdd("field"); fi.setIndexOptions(TextField.TYPE_STORED.indexOptions()); addAttributes(fi); @@ -221,7 +221,7 @@ public abstract class BaseFieldInfoFormatTestCase extends BaseIndexFileFormatTes dir.failOn(fail); Codec codec = getCodec(); SegmentInfo segmentInfo = newSegmentInfo(dir, "_123"); - FieldInfos.Builder builder = new FieldInfos.Builder(); + FieldInfos.Builder builder = new FieldInfos.Builder(new FieldInfos.FieldNumbers(null)); FieldInfo fi = builder.getOrAdd("field"); fi.setIndexOptions(TextField.TYPE_STORED.indexOptions()); addAttributes(fi); @@ -251,7 +251,7 @@ public abstract class BaseFieldInfoFormatTestCase extends BaseIndexFileFormatTes for (int i = 0; i < numFields; i++) { fieldNames.add(TestUtil.randomUnicodeString(random())); } - FieldInfos.Builder builder = new FieldInfos.Builder(); + FieldInfos.Builder builder = new FieldInfos.Builder(new FieldInfos.FieldNumbers(null)); for (String field : fieldNames) { IndexableFieldType fieldType = randomFieldType(random()); FieldInfo fi = builder.getOrAdd(field); http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/67b6593e/lucene/test-framework/src/java/org/apache/lucene/index/BaseIndexFileFormatTestCase.java ---------------------------------------------------------------------- diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/BaseIndexFileFormatTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/index/BaseIndexFileFormatTestCase.java index ab92946..f449644 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/index/BaseIndexFileFormatTestCase.java +++ b/lucene/test-framework/src/java/org/apache/lucene/index/BaseIndexFileFormatTestCase.java @@ -323,7 +323,7 @@ abstract class BaseIndexFileFormatTestCase extends LuceneTestCase { FieldInfo proto = oneDocReader.getFieldInfos().fieldInfo("field"); FieldInfo field = new FieldInfo(proto.name, proto.number, proto.hasVectors(), proto.omitsNorms(), proto.hasPayloads(), proto.getIndexOptions(), proto.getDocValuesType(), proto.getDocValuesGen(), new HashMap<>(), - proto.getPointDimensionCount(), proto.getPointNumBytes()); + proto.getPointDimensionCount(), proto.getPointNumBytes(), proto.isSoftDeletesField()); FieldInfos fieldInfos = new FieldInfos(new FieldInfo[] { field } ); http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/67b6593e/lucene/test-framework/src/java/org/apache/lucene/index/MismatchedLeafReader.java ---------------------------------------------------------------------- diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/MismatchedLeafReader.java b/lucene/test-framework/src/java/org/apache/lucene/index/MismatchedLeafReader.java index 7dd6ba8..2c74677 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/index/MismatchedLeafReader.java +++ b/lucene/test-framework/src/java/org/apache/lucene/index/MismatchedLeafReader.java @@ -77,7 +77,8 @@ public class MismatchedLeafReader extends FilterLeafReader { oldInfo.getDocValuesGen(), // dvGen oldInfo.attributes(), // attributes oldInfo.getPointDimensionCount(), // dimension count - oldInfo.getPointNumBytes()); // dimension numBytes + oldInfo.getPointNumBytes(), // dimension numBytes + oldInfo.isSoftDeletesField()); // used as soft-deletes field shuffled.set(i, newInfo); } http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/67b6593e/lucene/test-framework/src/java/org/apache/lucene/index/RandomPostingsTester.java ---------------------------------------------------------------------- diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/RandomPostingsTester.java b/lucene/test-framework/src/java/org/apache/lucene/index/RandomPostingsTester.java index d5eb105..0afab1c 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/index/RandomPostingsTester.java +++ b/lucene/test-framework/src/java/org/apache/lucene/index/RandomPostingsTester.java @@ -122,7 +122,7 @@ public class RandomPostingsTester { fieldInfoArray[fieldUpto] = new FieldInfo(field, fieldUpto, false, false, true, IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS, DocValuesType.NONE, -1, new HashMap<>(), - 0, 0); + 0, 0, false); fieldUpto++; SortedMap postings = new TreeMap<>(); @@ -638,7 +638,7 @@ public class RandomPostingsTester { DocValuesType.NONE, -1, new HashMap<>(), - 0, 0); + 0, 0, false); } FieldInfos newFieldInfos = new FieldInfos(newFieldInfoArray); http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/67b6593e/solr/core/src/java/org/apache/solr/handler/component/ExpandComponent.java ---------------------------------------------------------------------- diff --git a/solr/core/src/java/org/apache/solr/handler/component/ExpandComponent.java b/solr/core/src/java/org/apache/solr/handler/component/ExpandComponent.java index b3f792f..1dd5818 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/ExpandComponent.java +++ b/solr/core/src/java/org/apache/solr/handler/component/ExpandComponent.java @@ -796,7 +796,8 @@ public class ExpandComponent extends SearchComponent implements PluginInfoInitia fieldInfo.getDocValuesGen(), fieldInfo.attributes(), fieldInfo.getPointDimensionCount(), - fieldInfo.getPointNumBytes()); + fieldInfo.getPointNumBytes(), + fieldInfo.isSoftDeletesField()); newInfos.add(f); } else { http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/67b6593e/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java ---------------------------------------------------------------------- diff --git a/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java index 559ee8b..07bd2be 100644 --- a/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java @@ -424,7 +424,7 @@ public class CollapsingQParserPlugin extends QParserPlugin { DocValuesType.NONE, fieldInfo.getDocValuesGen(), fieldInfo.attributes(), - 0, 0); + 0, 0, fieldInfo.isSoftDeletesField()); newInfos.add(f); } else { http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/67b6593e/solr/core/src/java/org/apache/solr/search/Insanity.java ---------------------------------------------------------------------- diff --git a/solr/core/src/java/org/apache/solr/search/Insanity.java b/solr/core/src/java/org/apache/solr/search/Insanity.java index aa36652..8fe081f 100644 --- a/solr/core/src/java/org/apache/solr/search/Insanity.java +++ b/solr/core/src/java/org/apache/solr/search/Insanity.java @@ -66,7 +66,7 @@ public class Insanity { if (fi.name.equals(insaneField)) { filteredInfos.add(new FieldInfo(fi.name, fi.number, fi.hasVectors(), fi.omitsNorms(), fi.hasPayloads(), fi.getIndexOptions(), DocValuesType.NONE, -1, Collections.emptyMap(), - fi.getPointDimensionCount(), fi.getPointNumBytes())); + fi.getPointDimensionCount(), fi.getPointNumBytes(), fi.isSoftDeletesField())); } else { filteredInfos.add(fi); } http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/67b6593e/solr/core/src/java/org/apache/solr/uninverting/UninvertingReader.java ---------------------------------------------------------------------- diff --git a/solr/core/src/java/org/apache/solr/uninverting/UninvertingReader.java b/solr/core/src/java/org/apache/solr/uninverting/UninvertingReader.java index 967db54..9f0f527 100644 --- a/solr/core/src/java/org/apache/solr/uninverting/UninvertingReader.java +++ b/solr/core/src/java/org/apache/solr/uninverting/UninvertingReader.java @@ -282,7 +282,7 @@ public class UninvertingReader extends FilterLeafReader { } filteredInfos.add(new FieldInfo(fi.name, fi.number, fi.hasVectors(), fi.omitsNorms(), fi.hasPayloads(), fi.getIndexOptions(), type, fi.getDocValuesGen(), fi.attributes(), - fi.getPointDimensionCount(), fi.getPointNumBytes())); + fi.getPointDimensionCount(), fi.getPointNumBytes(), fi.isSoftDeletesField())); } fieldInfos = new FieldInfos(filteredInfos.toArray(new FieldInfo[filteredInfos.size()])); }