Return-Path: X-Original-To: apmail-lucene-commits-archive@www.apache.org Delivered-To: apmail-lucene-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 6476F102DD for ; Tue, 2 Dec 2014 16:45:12 +0000 (UTC) Received: (qmail 91289 invoked by uid 500); 2 Dec 2014 16:45:12 -0000 Mailing-List: contact commits-help@lucene.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@lucene.apache.org Delivered-To: mailing list commits@lucene.apache.org Received: (qmail 91279 invoked by uid 99); 2 Dec 2014 16:45:12 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 02 Dec 2014 16:45:12 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 02 Dec 2014 16:44:45 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 67CB72388E49; Tue, 2 Dec 2014 16:44:42 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1642943 - in /lucene/dev/branches/branch_5x: ./ lucene/ lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/ lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene46/ lucene/backward-codecs/src/test/org/apache/lucene... Date: Tue, 02 Dec 2014 16:44:41 -0000 To: commits@lucene.apache.org From: rmuir@apache.org X-Mailer: svnmailer-1.0.9 Message-Id: <20141202164442.67CB72388E49@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: rmuir Date: Tue Dec 2 16:44:40 2014 New Revision: 1642943 URL: http://svn.apache.org/r1642943 Log: LUCENE-6085: add back SI.attributes Modified: lucene/dev/branches/branch_5x/ (props changed) lucene/dev/branches/branch_5x/lucene/ (props changed) lucene/dev/branches/branch_5x/lucene/CHANGES.txt (contents, props changed) lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40SegmentInfoFormat.java lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene46/Lucene46SegmentInfoFormat.java lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene40/Lucene40RWSegmentInfoFormat.java lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene46/TestLucene46SegmentInfoFormat.java lucene/dev/branches/branch_5x/lucene/codecs/ (props changed) lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSegmentInfoFormat.java lucene/dev/branches/branch_5x/lucene/core/ (props changed) lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50SegmentInfoFormat.java lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/SegmentInfo.java lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestCodecs.java lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestDoc.java lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestSegmentMerger.java lucene/dev/branches/branch_5x/lucene/misc/ (props changed) lucene/dev/branches/branch_5x/lucene/misc/src/java/org/apache/lucene/index/IndexSplitter.java lucene/dev/branches/branch_5x/lucene/test-framework/ (props changed) lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/index/BaseCompoundFormatTestCase.java lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/index/BaseFieldInfoFormatTestCase.java lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/index/BasePostingsFormatTestCase.java lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/index/BaseSegmentInfoFormatTestCase.java Modified: lucene/dev/branches/branch_5x/lucene/CHANGES.txt URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/CHANGES.txt?rev=1642943&r1=1642942&r2=1642943&view=diff ============================================================================== --- lucene/dev/branches/branch_5x/lucene/CHANGES.txt (original) +++ lucene/dev/branches/branch_5x/lucene/CHANGES.txt Tue Dec 2 16:44:40 2014 @@ -211,6 +211,10 @@ API Changes * LUCENE-6087: Allow passing custom DirectoryReader to SearcherManager (Mike McCandless) +* LUCENE-6085: Undeprecate SegmentInfo attributes, but add safety so they + won't be trappy if codec tries to use them during docvalues updates. + (Robert Muir) + Bug Fixes * LUCENE-5650: Enforce read-only access to any path outside the temporary Modified: lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40SegmentInfoFormat.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40SegmentInfoFormat.java?rev=1642943&r1=1642942&r2=1642943&view=diff ============================================================================== --- lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40SegmentInfoFormat.java (original) +++ lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40SegmentInfoFormat.java Tue Dec 2 16:44:40 2014 @@ -19,6 +19,7 @@ package org.apache.lucene.codecs.lucene4 import java.io.IOException; import java.text.ParseException; +import java.util.Collections; import java.util.Map; import java.util.Set; @@ -65,12 +66,12 @@ public class Lucene40SegmentInfoFormat e } final boolean isCompoundFile = input.readByte() == SegmentInfo.YES; final Map diagnostics = input.readStringStringMap(); - input.readStringStringMap(); // read deprecated attributes + final Map attributes = input.readStringStringMap(); final Set files = input.readStringSet(); CodecUtil.checkEOF(input); - final SegmentInfo si = new SegmentInfo(dir, version, segment, docCount, isCompoundFile, null, diagnostics, null); + final SegmentInfo si = new SegmentInfo(dir, version, segment, docCount, isCompoundFile, null, diagnostics, null, Collections.unmodifiableMap(attributes)); si.setFiles(files); success = true; Modified: lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene46/Lucene46SegmentInfoFormat.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene46/Lucene46SegmentInfoFormat.java?rev=1642943&r1=1642942&r2=1642943&view=diff ============================================================================== --- lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene46/Lucene46SegmentInfoFormat.java (original) +++ lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene46/Lucene46SegmentInfoFormat.java Tue Dec 2 16:44:40 2014 @@ -19,6 +19,7 @@ package org.apache.lucene.codecs.lucene4 import java.io.IOException; import java.text.ParseException; +import java.util.Collections; import java.util.Map; import java.util.Set; @@ -71,7 +72,7 @@ public class Lucene46SegmentInfoFormat e CodecUtil.checkEOF(input); } - final SegmentInfo si = new SegmentInfo(dir, version, segment, docCount, isCompoundFile, null, diagnostics, null); + final SegmentInfo si = new SegmentInfo(dir, version, segment, docCount, isCompoundFile, null, diagnostics, null, Collections.emptyMap()); si.setFiles(files); return si; Modified: lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene40/Lucene40RWSegmentInfoFormat.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene40/Lucene40RWSegmentInfoFormat.java?rev=1642943&r1=1642942&r2=1642943&view=diff ============================================================================== --- lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene40/Lucene40RWSegmentInfoFormat.java (original) +++ lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene40/Lucene40RWSegmentInfoFormat.java Tue Dec 2 16:44:40 2014 @@ -51,7 +51,7 @@ public final class Lucene40RWSegmentInfo output.writeByte((byte) (si.getUseCompoundFile() ? SegmentInfo.YES : SegmentInfo.NO)); output.writeStringStringMap(si.getDiagnostics()); - output.writeStringStringMap(Collections.emptyMap()); + output.writeStringStringMap(si.getAttributes()); output.writeStringSet(si.files()); success = true; Modified: lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene46/TestLucene46SegmentInfoFormat.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene46/TestLucene46SegmentInfoFormat.java?rev=1642943&r1=1642942&r2=1642943&view=diff ============================================================================== --- lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene46/TestLucene46SegmentInfoFormat.java (original) +++ lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene46/TestLucene46SegmentInfoFormat.java Tue Dec 2 16:44:40 2014 @@ -17,6 +17,8 @@ package org.apache.lucene.codecs.lucene4 * limitations under the License. */ +import java.util.Map; + import org.apache.lucene.codecs.Codec; import org.apache.lucene.index.BaseSegmentInfoFormatTestCase; import org.apache.lucene.util.Version; @@ -51,6 +53,12 @@ public class TestLucene46SegmentInfoForm } @Override + @Deprecated + protected void assertAttributesEquals(Map expected, Map actual) { + assertEquals(0, actual.size()); // we don't support attributes + } + + @Override protected Codec getCodec() { return new Lucene46RWCodec(); } Modified: lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSegmentInfoFormat.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSegmentInfoFormat.java?rev=1642943&r1=1642942&r2=1642943&view=diff ============================================================================== --- lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSegmentInfoFormat.java (original) +++ lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSegmentInfoFormat.java Tue Dec 2 16:44:40 2014 @@ -21,6 +21,7 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; import java.text.ParseException; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.Map; @@ -52,6 +53,9 @@ public class SimpleTextSegmentInfoFormat final static BytesRef SI_NUM_DIAG = new BytesRef(" diagnostics "); final static BytesRef SI_DIAG_KEY = new BytesRef(" key "); final static BytesRef SI_DIAG_VALUE = new BytesRef(" value "); + final static BytesRef SI_NUM_ATT = new BytesRef(" attributes "); + final static BytesRef SI_ATT_KEY = new BytesRef(" key "); + final static BytesRef SI_ATT_VALUE = new BytesRef(" value "); final static BytesRef SI_NUM_FILES = new BytesRef(" files "); final static BytesRef SI_FILE = new BytesRef(" file "); final static BytesRef SI_ID = new BytesRef(" id "); @@ -97,6 +101,22 @@ public class SimpleTextSegmentInfoFormat } SimpleTextUtil.readLine(input, scratch); + assert StringHelper.startsWith(scratch.get(), SI_NUM_ATT); + int numAtt = Integer.parseInt(readString(SI_NUM_ATT.length, scratch)); + Map attributes = new HashMap<>(numAtt); + + for (int i = 0; i < numAtt; i++) { + SimpleTextUtil.readLine(input, scratch); + assert StringHelper.startsWith(scratch.get(), SI_ATT_KEY); + String key = readString(SI_ATT_KEY.length, scratch); + + SimpleTextUtil.readLine(input, scratch); + assert StringHelper.startsWith(scratch.get(), SI_ATT_VALUE); + String value = readString(SI_ATT_VALUE.length, scratch); + attributes.put(key, value); + } + + SimpleTextUtil.readLine(input, scratch); assert StringHelper.startsWith(scratch.get(), SI_NUM_FILES); int numFiles = Integer.parseInt(readString(SI_NUM_FILES.length, scratch)); Set files = new HashSet<>(); @@ -120,7 +140,7 @@ public class SimpleTextSegmentInfoFormat SimpleTextUtil.checkFooter(input); SegmentInfo info = new SegmentInfo(directory, version, segmentName, docCount, - isCompoundFile, null, diagnostics, id); + isCompoundFile, null, diagnostics, id, Collections.unmodifiableMap(attributes)); info.setFiles(files); return info; } @@ -169,6 +189,21 @@ public class SimpleTextSegmentInfoFormat } } + Map attributes = si.getAttributes(); + SimpleTextUtil.write(output, SI_NUM_ATT); + SimpleTextUtil.write(output, Integer.toString(attributes.size()), scratch); + SimpleTextUtil.writeNewline(output); + + for (Map.Entry attEntry : attributes.entrySet()) { + SimpleTextUtil.write(output, SI_ATT_KEY); + SimpleTextUtil.write(output, attEntry.getKey(), scratch); + SimpleTextUtil.writeNewline(output); + + SimpleTextUtil.write(output, SI_ATT_VALUE); + SimpleTextUtil.write(output, attEntry.getValue(), scratch); + SimpleTextUtil.writeNewline(output); + } + Set files = si.files(); int numFiles = files == null ? 0 : files.size(); SimpleTextUtil.write(output, SI_NUM_FILES); Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50SegmentInfoFormat.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50SegmentInfoFormat.java?rev=1642943&r1=1642942&r2=1642943&view=diff ============================================================================== --- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50SegmentInfoFormat.java (original) +++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50SegmentInfoFormat.java Tue Dec 2 16:44:40 2014 @@ -18,6 +18,7 @@ package org.apache.lucene.codecs.lucene5 */ import java.io.IOException; +import java.util.Collections; import java.util.Map; import java.util.Set; @@ -33,7 +34,6 @@ import org.apache.lucene.store.DataOutpu import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexOutput; -import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.Version; /** @@ -41,7 +41,7 @@ import org.apache.lucene.util.Version; *

* Files: *

    - *
  • .si: Header, SegVersion, SegSize, IsCompoundFile, Diagnostics, Files, Footer + *
  • .si: Header, SegVersion, SegSize, IsCompoundFile, Diagnostics, Files, Attributes, Footer *
*

* Data types: @@ -51,7 +51,7 @@ import org.apache.lucene.util.Version; *
  • SegSize --> {@link DataOutput#writeInt Int32}
    • *
  • SegVersion --> {@link DataOutput#writeString String}
    • *
  • Files --> {@link DataOutput#writeStringSet Set<String>}
    • - *
  • Diagnostics --> {@link DataOutput#writeStringStringMap Map<String,String>}
    • + *
  • Diagnostics,Attributes --> {@link DataOutput#writeStringStringMap Map<String,String>}
    • *
  • IsCompoundFile --> {@link DataOutput#writeByte Int8}
    • *
  • Footer --> {@link CodecUtil#writeFooter CodecFooter}
    • * @@ -101,8 +101,9 @@ public class Lucene50SegmentInfoFormat e final boolean isCompoundFile = input.readByte() == SegmentInfo.YES; final Map diagnostics = input.readStringStringMap(); final Set files = input.readStringSet(); + final Map attributes = input.readStringStringMap(); - si = new SegmentInfo(dir, version, segment, docCount, isCompoundFile, null, diagnostics, segmentID); + si = new SegmentInfo(dir, version, segment, docCount, isCompoundFile, null, diagnostics, segmentID, Collections.unmodifiableMap(attributes)); si.setFiles(files); } catch (Throwable exception) { priorE = exception; @@ -144,6 +145,7 @@ public class Lucene50SegmentInfoFormat e } } output.writeStringSet(files); + output.writeStringStringMap(si.getAttributes()); CodecUtil.writeFooter(output); } } Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java?rev=1642943&r1=1642942&r2=1642943&view=diff ============================================================================== --- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java (original) +++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java Tue Dec 2 16:44:40 2014 @@ -19,6 +19,7 @@ package org.apache.lucene.index; import java.io.IOException; import java.text.NumberFormat; +import java.util.HashMap; import java.util.HashSet; import java.util.Locale; import java.util.Set; @@ -178,7 +179,7 @@ class DocumentsWriterPerThread { pendingUpdates.clear(); deleteSlice = deleteQueue.newSlice(); - segmentInfo = new SegmentInfo(directoryOrig, Version.LATEST, segmentName, -1, false, codec, null, StringHelper.randomId()); + segmentInfo = new SegmentInfo(directoryOrig, Version.LATEST, segmentName, -1, false, codec, null, StringHelper.randomId(), new HashMap()); assert numDocsInRAM == 0; if (INFO_VERBOSE && infoStream.isEnabled("DWPT")) { infoStream.message("DWPT", Thread.currentThread().getName() + " init seg=" + segmentName + " delQueue=" + deleteQueue); Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java?rev=1642943&r1=1642942&r2=1642943&view=diff ============================================================================== --- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java (original) +++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java Tue Dec 2 16:44:40 2014 @@ -2540,7 +2540,7 @@ public class IndexWriter implements Clos TrackingDirectoryWrapper trackingDir = new TrackingDirectoryWrapper(directory); SegmentInfo info = new SegmentInfo(directory, Version.LATEST, mergedName, -1, - false, codec, null, StringHelper.randomId()); + false, codec, null, StringHelper.randomId(), new HashMap()); SegmentMerger merger = new SegmentMerger(mergeReaders, info, infoStream, trackingDir, MergeState.CheckAbort.NONE, globalFieldNumberMap, @@ -2636,7 +2636,7 @@ public class IndexWriter implements Clos // Same SI as before but we change directory and name SegmentInfo newInfo = new SegmentInfo(directory, info.info.getVersion(), segName, info.info.getDocCount(), info.info.getUseCompoundFile(), info.info.getCodec(), - info.info.getDiagnostics(), info.info.getId()); + info.info.getDiagnostics(), info.info.getId(), info.info.getAttributes()); SegmentCommitInfo newInfoPerCommit = new SegmentCommitInfo(newInfo, info.getDelCount(), info.getDelGen(), info.getFieldInfosGen(), info.getDocValuesGen()); @@ -3747,7 +3747,7 @@ public class IndexWriter implements Clos // ConcurrentMergePolicy we keep deterministic segment // names. final String mergeSegmentName = newSegmentName(); - SegmentInfo si = new SegmentInfo(directory, Version.LATEST, mergeSegmentName, -1, false, codec, null, StringHelper.randomId()); + SegmentInfo si = new SegmentInfo(directory, Version.LATEST, mergeSegmentName, -1, false, codec, null, StringHelper.randomId(), new HashMap()); Map details = new HashMap<>(); details.put("mergeMaxNumSegments", "" + merge.maxNumSegments); details.put("mergeFactor", Integer.toString(merge.segments.size())); Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/SegmentInfo.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/SegmentInfo.java?rev=1642943&r1=1642942&r2=1642943&view=diff ============================================================================== --- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/SegmentInfo.java (original) +++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/SegmentInfo.java Tue Dec 2 16:44:40 2014 @@ -21,8 +21,10 @@ package org.apache.lucene.index; import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.regex.Matcher; @@ -65,6 +67,8 @@ public final class SegmentInfo { private Codec codec; private Map diagnostics; + + private Map attributes; // Tracks the Lucene version this segment was created with, since 3.1. Null // indicates an older than 3.0 index, and it's used to detect a too old index. @@ -90,7 +94,7 @@ public final class SegmentInfo { */ public SegmentInfo(Directory dir, Version version, String name, int docCount, boolean isCompoundFile, Codec codec, Map diagnostics, - byte[] id) { + byte[] id, Map attributes) { assert !(dir instanceof TrackingDirectoryWrapper); this.dir = dir; this.version = version; @@ -103,6 +107,7 @@ public final class SegmentInfo { if (id != null && id.length != StringHelper.ID_LENGTH) { throw new IllegalArgumentException("invalid id: " + Arrays.toString(id)); } + this.attributes = Objects.requireNonNull(attributes); } /** @@ -267,4 +272,34 @@ public final class SegmentInfo { String namedForThisSegment(String file) { return name + IndexFileNames.stripSegmentName(file); } + + /** + * Get a codec attribute value, or null if it does not exist + */ + public String getAttribute(String key) { + return attributes.get(key); + } + + /** + * Puts a codec attribute value. + *

    + * This is a key-value mapping for the field that the codec can use to store + * additional metadata, and will be available to the codec when reading the + * segment via {@link #getAttribute(String)} + *

    + * If a value already exists for the field, it will be replaced with the new + * value. + */ + public String putAttribute(String key, String value) { + return attributes.put(key, value); + } + + /** + * Returns the internal codec attributes map. + * @return internal codec attributes map. + */ + public Map getAttributes() { + return attributes; + } } + Modified: lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestCodecs.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestCodecs.java?rev=1642943&r1=1642942&r2=1642943&view=diff ============================================================================== --- lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestCodecs.java (original) +++ lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestCodecs.java Tue Dec 2 16:44:40 2014 @@ -19,6 +19,7 @@ package org.apache.lucene.index; import java.io.IOException; import java.util.Arrays; +import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.Random; @@ -220,7 +221,7 @@ public class TestCodecs extends LuceneTe final FieldInfos fieldInfos = builder.finish(); final Directory dir = newDirectory(); Codec codec = Codec.getDefault(); - final SegmentInfo si = new SegmentInfo(dir, Version.LATEST, SEGMENT, 10000, false, codec, null, StringHelper.randomId()); + final SegmentInfo si = new SegmentInfo(dir, Version.LATEST, SEGMENT, 10000, false, codec, null, StringHelper.randomId(), new HashMap()); this.write(si, fieldInfos, dir, fields); final FieldsProducer reader = codec.postingsFormat().fieldsProducer(new SegmentReadState(dir, si, fieldInfos, newIOContext(random()))); @@ -277,7 +278,7 @@ public class TestCodecs extends LuceneTe } Codec codec = Codec.getDefault(); - final SegmentInfo si = new SegmentInfo(dir, Version.LATEST, SEGMENT, 10000, false, codec, null, StringHelper.randomId()); + final SegmentInfo si = new SegmentInfo(dir, Version.LATEST, SEGMENT, 10000, false, codec, null, StringHelper.randomId(), new HashMap()); this.write(si, fieldInfos, dir, fields); if (VERBOSE) { Modified: lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestDoc.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestDoc.java?rev=1642943&r1=1642942&r2=1642943&view=diff ============================================================================== --- lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestDoc.java (original) +++ lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestDoc.java Tue Dec 2 16:44:40 2014 @@ -28,6 +28,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.Arrays; import java.util.Collection; +import java.util.HashMap; import java.util.HashSet; import java.util.LinkedList; @@ -220,7 +221,7 @@ public class TestDoc extends LuceneTestC final Codec codec = Codec.getDefault(); TrackingDirectoryWrapper trackingDir = new TrackingDirectoryWrapper(si1.info.dir); - final SegmentInfo si = new SegmentInfo(si1.info.dir, Version.LATEST, merged, -1, false, codec, null, StringHelper.randomId()); + final SegmentInfo si = new SegmentInfo(si1.info.dir, Version.LATEST, merged, -1, false, codec, null, StringHelper.randomId(), new HashMap()); SegmentMerger merger = new SegmentMerger(Arrays.asList(r1, r2), si, InfoStream.getDefault(), trackingDir, Modified: lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestSegmentMerger.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestSegmentMerger.java?rev=1642943&r1=1642942&r2=1642943&view=diff ============================================================================== --- lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestSegmentMerger.java (original) +++ lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestSegmentMerger.java Tue Dec 2 16:44:40 2014 @@ -19,6 +19,7 @@ package org.apache.lucene.index; import java.io.IOException; import java.util.Arrays; +import java.util.HashMap; import org.apache.lucene.codecs.Codec; import org.apache.lucene.document.Document; @@ -79,7 +80,7 @@ public class TestSegmentMerger extends L public void testMerge() throws IOException { final Codec codec = Codec.getDefault(); - final SegmentInfo si = new SegmentInfo(mergedDir, Version.LATEST, mergedSegment, -1, false, codec, null, StringHelper.randomId()); + final SegmentInfo si = new SegmentInfo(mergedDir, Version.LATEST, mergedSegment, -1, false, codec, null, StringHelper.randomId(), new HashMap()); SegmentMerger merger = new SegmentMerger(Arrays.asList(reader1, reader2), si, InfoStream.getDefault(), mergedDir, Modified: lucene/dev/branches/branch_5x/lucene/misc/src/java/org/apache/lucene/index/IndexSplitter.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/misc/src/java/org/apache/lucene/index/IndexSplitter.java?rev=1642943&r1=1642942&r2=1642943&view=diff ============================================================================== --- lucene/dev/branches/branch_5x/lucene/misc/src/java/org/apache/lucene/index/IndexSplitter.java (original) +++ lucene/dev/branches/branch_5x/lucene/misc/src/java/org/apache/lucene/index/IndexSplitter.java Tue Dec 2 16:44:40 2014 @@ -24,6 +24,7 @@ import java.text.DecimalFormat; import java.text.DecimalFormatSymbols; import java.util.ArrayList; import java.util.Collection; +import java.util.HashMap; import java.util.List; import java.util.Locale; @@ -137,7 +138,7 @@ public class IndexSplitter { SegmentInfo info = infoPerCommit.info; // Same info just changing the dir: SegmentInfo newInfo = new SegmentInfo(destFSDir, info.getVersion(), info.name, info.getDocCount(), - info.getUseCompoundFile(), info.getCodec(), info.getDiagnostics(), info.getId()); + info.getUseCompoundFile(), info.getCodec(), info.getDiagnostics(), info.getId(), new HashMap()); destInfos.add(new SegmentCommitInfo(newInfo, infoPerCommit.getDelCount(), infoPerCommit.getDelGen(), infoPerCommit.getFieldInfosGen(), infoPerCommit.getDocValuesGen())); Modified: lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/index/BaseCompoundFormatTestCase.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/index/BaseCompoundFormatTestCase.java?rev=1642943&r1=1642942&r2=1642943&view=diff ============================================================================== --- lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/index/BaseCompoundFormatTestCase.java (original) +++ lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/index/BaseCompoundFormatTestCase.java Tue Dec 2 16:44:40 2014 @@ -21,6 +21,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.List; import org.apache.lucene.codecs.Codec; @@ -626,7 +627,7 @@ public abstract class BaseCompoundFormat /** Returns a new fake segment */ protected static SegmentInfo newSegmentInfo(Directory dir, String name) { - return new SegmentInfo(dir, Version.LATEST, name, 10000, false, Codec.getDefault(), null, StringHelper.randomId()); + return new SegmentInfo(dir, Version.LATEST, name, 10000, false, Codec.getDefault(), null, StringHelper.randomId(), new HashMap()); } /** Creates a file of the specified size with random data. */ Modified: lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/index/BaseFieldInfoFormatTestCase.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/index/BaseFieldInfoFormatTestCase.java?rev=1642943&r1=1642942&r2=1642943&view=diff ============================================================================== --- lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/index/BaseFieldInfoFormatTestCase.java (original) +++ lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/index/BaseFieldInfoFormatTestCase.java Tue Dec 2 16:44:40 2014 @@ -18,6 +18,7 @@ package org.apache.lucene.index; */ import java.io.IOException; +import java.util.HashMap; import java.util.HashSet; import java.util.Random; import java.util.Set; @@ -175,7 +176,7 @@ public abstract class BaseFieldInfoForma /** Returns a new fake segment */ protected static SegmentInfo newSegmentInfo(Directory dir, String name) { - return new SegmentInfo(dir, Version.LATEST, name, 10000, false, Codec.getDefault(), null, StringHelper.randomId()); + return new SegmentInfo(dir, Version.LATEST, name, 10000, false, Codec.getDefault(), null, StringHelper.randomId(), new HashMap()); } @Override Modified: lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/index/BasePostingsFormatTestCase.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/index/BasePostingsFormatTestCase.java?rev=1642943&r1=1642942&r2=1642943&view=diff ============================================================================== --- lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/index/BasePostingsFormatTestCase.java (original) +++ lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/index/BasePostingsFormatTestCase.java Tue Dec 2 16:44:40 2014 @@ -23,6 +23,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.EnumSet; +import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.List; @@ -674,7 +675,7 @@ public abstract class BasePostingsFormat // randomly index at lower IndexOption private FieldsProducer buildIndex(Directory dir, IndexOptions maxAllowed, boolean allowPayloads, boolean alwaysTestMax) throws IOException { Codec codec = getCodec(); - SegmentInfo segmentInfo = new SegmentInfo(dir, Version.LATEST, "_0", maxDoc, false, codec, null, StringHelper.randomId()); + SegmentInfo segmentInfo = new SegmentInfo(dir, Version.LATEST, "_0", maxDoc, false, codec, null, StringHelper.randomId(), new HashMap()); int maxIndexOption = Arrays.asList(IndexOptions.values()).indexOf(maxAllowed); if (VERBOSE) { Modified: lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/index/BaseSegmentInfoFormatTestCase.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/index/BaseSegmentInfoFormatTestCase.java?rev=1642943&r1=1642942&r2=1642943&view=diff ============================================================================== --- lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/index/BaseSegmentInfoFormatTestCase.java (original) +++ lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/index/BaseSegmentInfoFormatTestCase.java Tue Dec 2 16:44:40 2014 @@ -49,7 +49,7 @@ public abstract class BaseSegmentInfoFor Codec codec = getCodec(); byte id[] = StringHelper.randomId(); SegmentInfo info = new SegmentInfo(dir, getVersions()[0], "_123", 1, false, codec, - Collections.emptyMap(), id); + Collections.emptyMap(), id, new HashMap()); info.setFiles(Collections.emptySet()); codec.segmentInfoFormat().write(dir, info, IOContext.DEFAULT); SegmentInfo info2 = codec.segmentInfoFormat().read(dir, "_123", id, IOContext.DEFAULT); @@ -63,7 +63,7 @@ public abstract class BaseSegmentInfoFor Codec codec = getCodec(); byte id[] = StringHelper.randomId(); SegmentInfo info = new SegmentInfo(dir, getVersions()[0], "_123", 1, false, codec, - Collections.emptyMap(), id); + Collections.emptyMap(), id, new HashMap()); Set originalFiles = Collections.singleton("_123.a"); info.setFiles(originalFiles); codec.segmentInfoFormat().write(dir, info, IOContext.DEFAULT); @@ -86,7 +86,7 @@ public abstract class BaseSegmentInfoFor diagnostics.put("key1", "value1"); diagnostics.put("key2", "value2"); SegmentInfo info = new SegmentInfo(dir, getVersions()[0], "_123", 1, false, codec, - diagnostics, id); + diagnostics, id, new HashMap()); info.setFiles(Collections.emptySet()); codec.segmentInfoFormat().write(dir, info, IOContext.DEFAULT); SegmentInfo info2 = codec.segmentInfoFormat().read(dir, "_123", id, IOContext.DEFAULT); @@ -94,13 +94,30 @@ public abstract class BaseSegmentInfoFor dir.close(); } + /** Test attributes map */ + public void testAttributes() throws Exception { + Directory dir = newDirectory(); + Codec codec = getCodec(); + byte id[] = StringHelper.randomId(); + Map attributes = new HashMap<>(); + attributes.put("key1", "value1"); + attributes.put("key2", "value2"); + SegmentInfo info = new SegmentInfo(dir, getVersions()[0], "_123", 1, false, codec, + Collections.emptyMap(), id, attributes); + info.setFiles(Collections.emptySet()); + codec.segmentInfoFormat().write(dir, info, IOContext.DEFAULT); + SegmentInfo info2 = codec.segmentInfoFormat().read(dir, "_123", id, IOContext.DEFAULT); + assertAttributesEquals(attributes, info2.getAttributes()); + dir.close(); + } + /** Test unique ID */ public void testUniqueID() throws Exception { Codec codec = getCodec(); Directory dir = newDirectory(); byte id[] = StringHelper.randomId(); SegmentInfo info = new SegmentInfo(dir, getVersions()[0], "_123", 1, false, codec, - Collections.emptyMap(), id); + Collections.emptyMap(), id, new HashMap()); info.setFiles(Collections.emptySet()); codec.segmentInfoFormat().write(dir, info, IOContext.DEFAULT); SegmentInfo info2 = codec.segmentInfoFormat().read(dir, "_123", id, IOContext.DEFAULT); @@ -115,7 +132,7 @@ public abstract class BaseSegmentInfoFor Directory dir = newDirectory(); byte id[] = StringHelper.randomId(); SegmentInfo info = new SegmentInfo(dir, v, "_123", 1, false, codec, - Collections.emptyMap(), id); + Collections.emptyMap(), id, new HashMap()); info.setFiles(Collections.emptySet()); codec.segmentInfoFormat().write(dir, info, IOContext.DEFAULT); SegmentInfo info2 = codec.segmentInfoFormat().read(dir, "_123", id, IOContext.DEFAULT); @@ -153,7 +170,14 @@ public abstract class BaseSegmentInfoFor byte id[] = new byte[StringHelper.ID_LENGTH]; random().nextBytes(id); - SegmentInfo info = new SegmentInfo(dir, version, name, docCount, isCompoundFile, codec, diagnostics, id); + Map attributes = new HashMap<>(); + int numAttributes = random().nextInt(10); + for (int j = 0; j < numAttributes; j++) { + attributes.put(TestUtil.randomUnicodeString(random()), + TestUtil.randomUnicodeString(random())); + } + + SegmentInfo info = new SegmentInfo(dir, version, name, docCount, isCompoundFile, codec, diagnostics, id, attributes); info.setFiles(files); codec.segmentInfoFormat().write(dir, info, IOContext.DEFAULT); SegmentInfo info2 = codec.segmentInfoFormat().read(dir, name, id, IOContext.DEFAULT); @@ -174,6 +198,7 @@ public abstract class BaseSegmentInfoFor assertIDEquals(expected.getId(), actual.getId()); assertEquals(expected.getUseCompoundFile(), actual.getUseCompoundFile()); assertEquals(expected.getVersion(), actual.getVersion()); + assertAttributesEquals(expected.getAttributes(), actual.getAttributes()); } /** Returns the versions this SI should test */ @@ -188,6 +213,15 @@ public abstract class BaseSegmentInfoFor assertArrayEquals(expected, actual); } + /** + * assert that attributes map is equal. + * @deprecated only exists to be overridden by old codecs that didnt support this + */ + @Deprecated + protected void assertAttributesEquals(Map expected, Map actual) { + assertEquals(expected, actual); + } + @Override protected void addRandomFields(Document doc) { doc.add(new StoredField("foobar", TestUtil.randomSimpleString(random())));