lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mikemcc...@apache.org
Subject svn commit: r1634823 [5/9] - in /lucene/dev/branches/lucene6005: ./ dev-tools/ lucene/ lucene/analysis/ lucene/analysis/common/ lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/ lucene/analysis/common/src/java/org/apache/lucene/analy...
Date Tue, 28 Oct 2014 09:19:29 GMT
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/FixedBitSet.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/FixedBitSet.java?rev=1634823&r1=1634822&r2=1634823&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/FixedBitSet.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/FixedBitSet.java Tue Oct 28 09:19:21 2014
@@ -31,71 +31,11 @@ import org.apache.lucene.search.DocIdSet
  * 
  * @lucene.internal
  */
-public final class FixedBitSet extends DocIdSet implements MutableBits {
+public final class FixedBitSet extends BitSet implements MutableBits, Accountable {
 
   private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(FixedBitSet.class);
 
   /**
-   * A {@link DocIdSetIterator} which iterates over set bits in a
-   * {@link FixedBitSet}.
-   */
-  public static final class FixedBitSetIterator extends DocIdSetIterator {
-    
-    final int numBits, numWords;
-    final long[] bits;
-    int doc = -1;
-    
-    /** Creates an iterator over the given {@link FixedBitSet}. */
-    public FixedBitSetIterator(FixedBitSet bits) {
-      this(bits.bits, bits.numBits, bits.numWords);
-    }
-    
-    /** Creates an iterator over the given array of bits. */
-    public FixedBitSetIterator(long[] bits, int numBits, int wordLength) {
-      this.bits = bits;
-      this.numBits = numBits;
-      this.numWords = wordLength;
-    }
-    
-    @Override
-    public int nextDoc() {
-      return advance(doc + 1);
-    }
-    
-    @Override
-    public int docID() {
-      return doc;
-    }
-    
-    @Override
-    public long cost() {
-      return numBits;
-    }
-    
-    @Override
-    public int advance(int target) {
-      if (target >= numBits) {
-        return doc = NO_MORE_DOCS;
-      }
-      int i = target >> 6;
-      long word = bits[i] >> target; // skip all the bits to the right of index
-      
-      if (word != 0) {
-        return doc = target + Long.numberOfTrailingZeros(word);
-      }
-      
-      while (++i < numWords) {
-        word = bits[i];
-        if (word != 0) {
-          return doc = (i << 6) + Long.numberOfTrailingZeros(word);
-        }
-      }
-      
-      return doc = NO_MORE_DOCS;
-    }
-  }
-
-  /**
    * If the given {@link FixedBitSet} is large enough to hold {@code numBits},
    * returns the given bits, otherwise returns a new {@link FixedBitSet} which
    * can hold the requested number of bits.
@@ -179,28 +119,12 @@ public final class FixedBitSet extends D
     this.numBits = numBits;
     this.bits = storedBits;
   }
-  
-  @Override
-  public DocIdSetIterator iterator() {
-    return new FixedBitSetIterator(bits, numBits, numWords);
-  }
-
-  @Override
-  public Bits bits() {
-    return this;
-  }
 
   @Override
   public int length() {
     return numBits;
   }
 
-  /** This DocIdSet implementation is cacheable. */
-  @Override
-  public boolean isCacheable() {
-    return true;
-  }
-
   @Override
   public long ramBytesUsed() {
     return BASE_RAM_BYTES_USED + RamUsageEstimator.sizeOf(bits);
@@ -211,9 +135,7 @@ public final class FixedBitSet extends D
     return bits;
   }
 
-  /** Returns number of set bits.  NOTE: this visits every
-   *  long in the backing bits array, and the result is not
-   *  internally cached! */
+  @Override
   public int cardinality() {
     return (int) BitUtil.pop_array(bits, 0, bits.length);
   }
@@ -244,6 +166,7 @@ public final class FixedBitSet extends D
     return val;
   }
 
+  @Override
   public void clear(int index) {
     assert index >= 0 && index < numBits;
     int wordNum = index >> 6;
@@ -260,9 +183,7 @@ public final class FixedBitSet extends D
     return val;
   }
 
-  /** Returns the index of the first set bit starting at the index specified.
-   *  -1 is returned if there are no more set bits.
-   */
+  @Override
   public int nextSetBit(int index) {
     assert index >= 0 && index < numBits : "index=" + index + ", numBits=" + numBits;
     int i = index >> 6;
@@ -279,7 +200,7 @@ public final class FixedBitSet extends D
       }
     }
 
-    return -1;
+    return DocIdSetIterator.NO_MORE_DOCS;
   }
 
   /** Returns the index of the last set bit before or on the index specified.
@@ -305,20 +226,14 @@ public final class FixedBitSet extends D
     return -1;
   }
 
-  /** Does in-place OR of the bits provided by the
-   *  iterator. */
+  @Override
   public void or(DocIdSetIterator iter) throws IOException {
-    if (iter instanceof FixedBitSetIterator && iter.docID() == -1) {
-      final FixedBitSetIterator fbs = (FixedBitSetIterator) iter;
-      or(fbs.bits, fbs.numWords);
-      // advance after last doc that would be accepted if standard
-      // iteration is used (to exhaust it):
-      fbs.advance(numBits);
+    if (BitSetIterator.getFixedBitSetOrNull(iter) != null) {
+      assertUnpositioned(iter);
+      final FixedBitSet bits = BitSetIterator.getFixedBitSetOrNull(iter); 
+      or(bits);
     } else {
-      int doc;
-      while ((doc = iter.nextDoc()) < numBits) {
-        set(doc);
-      }
+      super.or(iter);
     }
   }
 
@@ -338,43 +253,40 @@ public final class FixedBitSet extends D
   
   /** this = this XOR other */
   public void xor(FixedBitSet other) {
-    assert other.numWords <= numWords : "numWords=" + numWords + ", other.numWords=" + other.numWords;
-    final long[] thisBits = this.bits;
-    final long[] otherBits = other.bits;
-    int pos = Math.min(numWords, other.numWords);
-    while (--pos >= 0) {
-      thisBits[pos] ^= otherBits[pos];
-    }
+    xor(other.bits, other.numWords);
   }
   
   /** Does in-place XOR of the bits provided by the iterator. */
   public void xor(DocIdSetIterator iter) throws IOException {
-    int doc;
-    while ((doc = iter.nextDoc()) < numBits) {
-      flip(doc, doc + 1);
+    assertUnpositioned(iter);
+    if (BitSetIterator.getFixedBitSetOrNull(iter) != null) {
+      final FixedBitSet bits = BitSetIterator.getFixedBitSetOrNull(iter); 
+      xor(bits);
+    } else {
+      int doc;
+      while ((doc = iter.nextDoc()) < numBits) {
+        flip(doc);
+      }
     }
   }
 
-  /** Does in-place AND of the bits provided by the
-   *  iterator. */
+  private void xor(long[] otherBits, int otherNumWords) {
+    assert otherNumWords <= numWords : "numWords=" + numWords + ", other.numWords=" + otherNumWords;
+    final long[] thisBits = this.bits;
+    int pos = Math.min(numWords, otherNumWords);
+    while (--pos >= 0) {
+      thisBits[pos] ^= otherBits[pos];
+    }
+  }
+
+  @Override
   public void and(DocIdSetIterator iter) throws IOException {
-    if (iter instanceof FixedBitSetIterator && iter.docID() == -1) {
-      final FixedBitSetIterator fbs = (FixedBitSetIterator) iter;
-      and(fbs.bits, fbs.numWords);
-      // advance after last doc that would be accepted if standard
-      // iteration is used (to exhaust it):
-      fbs.advance(numBits);
+    if (BitSetIterator.getFixedBitSetOrNull(iter) != null) {
+      assertUnpositioned(iter);
+      final FixedBitSet bits = BitSetIterator.getFixedBitSetOrNull(iter); 
+      and(bits);
     } else {
-      if (numBits == 0) return;
-      int disiDoc, bitSetDoc = nextSetBit(0);
-      while (bitSetDoc != -1 && (disiDoc = iter.advance(bitSetDoc)) < numBits) {
-        clear(bitSetDoc, disiDoc);
-        disiDoc++;
-        bitSetDoc = (disiDoc < numBits) ? nextSetBit(disiDoc) : -1;
-      }
-      if (bitSetDoc != -1) {
-        clear(bitSetDoc, numBits);
-      }
+      super.and(iter);
     }
   }
 
@@ -403,20 +315,14 @@ public final class FixedBitSet extends D
     }
   }
 
-  /** Does in-place AND NOT of the bits provided by the
-   *  iterator. */
+  @Override
   public void andNot(DocIdSetIterator iter) throws IOException {
-    if (iter instanceof FixedBitSetIterator && iter.docID() == -1) {
-      final FixedBitSetIterator fbs = (FixedBitSetIterator) iter;
-      andNot(fbs.bits, fbs.numWords);
-      // advance after last doc that would be accepted if standard
-      // iteration is used (to exhaust it):
-      fbs.advance(numBits);
+    if (BitSetIterator.getFixedBitSetOrNull(iter) != null) {
+      assertUnpositioned(iter);
+      final FixedBitSet bits = BitSetIterator.getFixedBitSetOrNull(iter); 
+      andNot(bits);
     } else {
-      int doc;
-      while ((doc = iter.nextDoc()) < numBits) {
-        clear(doc);
-      }
+      super.andNot(iter);
     }
   }
 
@@ -476,6 +382,15 @@ public final class FixedBitSet extends D
     bits[endWord] ^= endmask;
   }
 
+  /** Flip the bit at the provided index. */
+  public void flip(int index) {
+    assert index >= 0 && index < numBits: "index=" + index + " numBits=" + numBits;
+    int wordNum = index >> 6;      // div 64
+    int bit = index & 0x3f;     // mod 64
+    long bitmask = 1L << bit;
+    bits[wordNum] ^= bitmask;
+  }
+
   /** Sets a range of bits
    *
    * @param startIndex lower index
@@ -504,11 +419,7 @@ public final class FixedBitSet extends D
     bits[endWord] |= endmask;
   }
 
-  /** Clears a range of bits.
-   *
-   * @param startIndex lower index
-   * @param endIndex one-past the last bit to clear
-   */
+  @Override
   public void clear(int startIndex, int endIndex) {
     assert startIndex >= 0 && startIndex < numBits : "startIndex=" + startIndex + ", numBits=" + numBits;
     assert endIndex >= 0 && endIndex <= numBits : "endIndex=" + endIndex + ", numBits=" + numBits;

Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/NamedSPILoader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/NamedSPILoader.java?rev=1634823&r1=1634822&r2=1634823&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/NamedSPILoader.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/NamedSPILoader.java Tue Oct 28 09:19:21 2014
@@ -106,9 +106,9 @@ public final class NamedSPILoader<S exte
   public S lookup(String name) {
     final S service = services.get(name);
     if (service != null) return service;
-    throw new IllegalArgumentException("A SPI class of type "+clazz.getName()+" with name '"+name+"' does not exist. "+
-     "You need to add the corresponding JAR file supporting this SPI to your classpath."+
-     "The current classpath supports the following names: "+availableServices());
+    throw new IllegalArgumentException("An SPI class of type "+clazz.getName()+" with name '"+name+"' does not exist."+
+     "  You need to add the corresponding JAR file supporting this SPI to your classpath."+
+     "  The current classpath supports the following names: "+availableServices());
   }
 
   public Set<String> availableServices() {

Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/RoaringDocIdSet.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/RoaringDocIdSet.java?rev=1634823&r1=1634822&r2=1634823&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/RoaringDocIdSet.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/RoaringDocIdSet.java Tue Oct 28 09:19:21 2014
@@ -85,14 +85,14 @@ public class RoaringDocIdSet extends Doc
           int excludedDoc = -1;
           for (int i = 0; i < excludedDocs.length; ++i) {
             excludedDoc = denseBuffer.nextSetBit(excludedDoc + 1);
-            assert excludedDoc != -1;
+            assert excludedDoc != DocIdSetIterator.NO_MORE_DOCS;
             excludedDocs[i] = (short) excludedDoc;
           }
-          assert excludedDoc + 1 == denseBuffer.length() || denseBuffer.nextSetBit(excludedDoc + 1) == -1;
+          assert excludedDoc + 1 == denseBuffer.length() || denseBuffer.nextSetBit(excludedDoc + 1) == DocIdSetIterator.NO_MORE_DOCS;
           sets[currentBlock] = new NotDocIdSet(BLOCK_SIZE, new ShortArrayDocIdSet(excludedDocs));
         } else {
           // Neither sparse nor super dense, use a fixed bit set
-          sets[currentBlock] = denseBuffer;
+          sets[currentBlock] = new BitDocIdSet(denseBuffer, currentBlockCardinality);
         }
         denseBuffer = null;
       }

Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/SPIClassIterator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/SPIClassIterator.java?rev=1634823&r1=1634822&r2=1634823&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/SPIClassIterator.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/SPIClassIterator.java Tue Oct 28 09:19:21 2014
@@ -140,7 +140,7 @@ public final class SPIClassIterator<S> i
       // don't initialize the class (pass false as 2nd parameter):
       return Class.forName(c, false, loader).asSubclass(clazz);
     } catch (ClassNotFoundException cnfe) {
-      throw new ServiceConfigurationError(String.format(Locale.ROOT, "A SPI class of type %s with classname %s does not exist, "+
+      throw new ServiceConfigurationError(String.format(Locale.ROOT, "An SPI class of type %s with classname %s does not exist, "+
         "please fix the file '%s%1$s' in your classpath.", clazz.getName(), c, META_INF_SERVICES));
     }
   }

Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/SparseFixedBitSet.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/SparseFixedBitSet.java?rev=1634823&r1=1634822&r2=1634823&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/SparseFixedBitSet.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/SparseFixedBitSet.java Tue Oct 28 09:19:21 2014
@@ -18,8 +18,8 @@ package org.apache.lucene.util;
  */
 
 import java.io.IOException;
+import java.util.Arrays;
 
-import org.apache.lucene.search.DocIdSet;
 import org.apache.lucene.search.DocIdSetIterator;
 
 /**
@@ -33,10 +33,11 @@ import org.apache.lucene.search.DocIdSet
  *
  * @lucene.internal
  */
-public class SparseFixedBitSet extends DocIdSet implements Bits {
+public class SparseFixedBitSet extends BitSet implements Bits, Accountable {
 
   private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(SparseFixedBitSet.class);
   private static final long SINGLE_ELEMENT_ARRAY_BYTES_USED = RamUsageEstimator.sizeOf(new long[1]);
+  private static final int MASK_4096 = (1 << 12) - 1;
 
   private static int blockCount(int length) {
     int blockCount = length >>> 12;
@@ -69,29 +70,16 @@ public class SparseFixedBitSet extends D
   }
 
   @Override
-  public boolean isCacheable() {
-    return true;
-  }
-
-  @Override
   public int length() {
     return length;
   }
 
-  @Override
-  public Bits bits() throws IOException {
-    return this;
-  }
-
   private boolean consistent(int index) {
     assert index >= 0 && index < length : "index=" + index + ",length=" + length;
     return true;
   }
 
-  /**
-   * Compute the cardinality of this set.
-   * NOTE: this operation runs in linear time.
-   */
+  @Override
   public int cardinality() {
     int cardinality = 0;
     for (long[] bitArray : bits) {
@@ -104,12 +92,11 @@ public class SparseFixedBitSet extends D
     return cardinality;
   }
 
-  /**
-   * Return an approximation of the cardinality of this set, assuming that bits
-   * are uniformly distributed. This operation runs in constant time.
-   */
+  @Override
   public int approximateCardinality() {
-    // this is basically the linear counting algorithm
+    // we are assuming that bits are uniformly set and use the linear counting
+    // algorithm to estimate the number of bits that are set based on the number
+    // of longs that are different from zero
     final int totalLongs = (length + 63) >>> 6; // total number of longs in the space
     assert totalLongs >= nonZeroLongCount;
     final int zeroLongs = totalLongs - nonZeroLongCount; // number of longs that are zeros
@@ -169,7 +156,7 @@ public class SparseFixedBitSet extends D
       insertLong(i4096, i64, i, index);
     }
   }
-  
+
   private void insertBlock(int i4096, int i64, int i) {
     indices[i4096] = 1L << i64; // shifts are mod 64 in java
     assert bits[i4096] == null;
@@ -203,128 +190,318 @@ public class SparseFixedBitSet extends D
   }
 
   /**
-   * Add the documents contained in the provided {@link DocIdSetIterator} to
-   * this bit set.
+   * Clear the bit at index <tt>i</tt>.
    */
-  public void or(DocIdSetIterator it) throws IOException {
-    for (int doc = it.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = it.nextDoc()) {
-      set(doc);
+  public void clear(int i) {
+    assert consistent(i);
+    final int i4096 = i >>> 12;
+    final int i64 = i >>> 6;
+    and(i4096, i64, ~(1L << i));
+  }
+
+  private void and(int i4096, int i64, long mask) {
+    final long index = indices[i4096];
+    if ((index & (1L << i64)) != 0) {
+      // offset of the long bits we are interested in in the array
+      final int o = Long.bitCount(index & ((1L << i64) - 1));
+      long bits = this.bits[i4096][o] & mask;
+      if (bits == 0) {
+        removeLong(i4096, i64, index, o);
+      } else {
+        this.bits[i4096][o] = bits;
+      }
     }
   }
 
-  @Override
-  public long ramBytesUsed() {
-    return ramBytesUsed;
+  private void removeLong(int i4096, int i64, long index, int o) {
+    index &= ~(1L << i64);
+    indices[i4096] = index;
+    if (index == 0) {
+      // release memory, there is nothing in this block anymore
+      this.bits[i4096] = null;
+    } else {
+      final int length = Long.bitCount(index);
+      final long[] bitArray = bits[i4096];
+      System.arraycopy(bitArray, o + 1, bitArray, o, length - o);
+      bitArray[length] = 0L;
+    }
+    nonZeroLongCount -= 1;
   }
 
   @Override
-  public DocIdSetIterator iterator() throws IOException {
-    return new Iterator();
+  public void clear(int from, int to) {
+    assert from >= 0;
+    assert to <= length;
+    if (from >= to) {
+      return;
+    }
+    final int firstBlock = from >>> 12;
+    final int lastBlock = (to - 1) >>> 12;
+    if (firstBlock == lastBlock) {
+      clearWithinBlock(firstBlock, from & MASK_4096, (to - 1) & MASK_4096);
+    } else {
+      clearWithinBlock(firstBlock, from & MASK_4096, MASK_4096);
+      for (int i = firstBlock + 1; i < lastBlock; ++i) {
+        nonZeroLongCount -= Long.bitCount(indices[i]);
+        indices[i] = 0;
+        bits[i] = null;
+      }
+      clearWithinBlock(lastBlock, 0, (to - 1) & MASK_4096);
+    }
   }
 
-  class Iterator extends DocIdSetIterator {
+  // create a long that has bits set to one between from and to
+  private static long mask(int from, int to) {
+    return ((1L << (to - from) << 1) - 1) << from;
+  }
 
-    private int doc = -1;
-    private int cost = -1;
+  private void clearWithinBlock(int i4096, int from, int to) {
+    int firstLong = from >>> 6;
+    int lastLong = to >>> 6;
 
-    @Override
-    public int docID() {
-      return doc;
+    if (firstLong == lastLong) {
+      and(i4096, firstLong, ~mask(from, to));
+    } else {
+      assert firstLong < lastLong;
+      and(i4096, lastLong, ~mask(0, to));
+      for (int i = lastLong - 1; i >= firstLong + 1; --i) {
+        and(i4096, i, 0L);
+      }
+      and(i4096, firstLong, ~mask(from, 63));
     }
+  }
 
-    /** Return the first document that occurs on or after the provided block index. */
-    private int firstDoc(int i4096) {
-      long index = 0;
-      while (i4096 < indices.length) {
-        index = indices[i4096];
-        if (index != 0) {
-          final int i64 = Long.numberOfTrailingZeros(index);
-          return doc = (i4096 << 12) | (i64 << 6) | Long.numberOfTrailingZeros(bits[i4096][0]);
-        }
-        i4096 += 1;
+  /** Return the first document that occurs on or after the provided block index. */
+  private int firstDoc(int i4096) {
+    long index = 0;
+    while (i4096 < indices.length) {
+      index = indices[i4096];
+      if (index != 0) {
+        final int i64 = Long.numberOfTrailingZeros(index);
+        return (i4096 << 12) | (i64 << 6) | Long.numberOfTrailingZeros(bits[i4096][0]);
       }
-      return doc = NO_MORE_DOCS;
+      i4096 += 1;
     }
+    return DocIdSetIterator.NO_MORE_DOCS;
+  }
 
-    @Override
-    public int nextDoc() throws IOException {
-      return advance(doc + 1);
-    }
+  @Override
+  public int nextSetBit(int i) {
+    assert i < length;
+    final int i4096 = i >>> 12;
+    final long index = indices[i4096];
+    int i64 = i >>> 6;
+    long indexBits = index >>> i64;
+    if (indexBits == 0) {
+      // if the index is zero, it means that there is no value in the
+      // current block, so return the first document of the next block
+      // or
+      // if neither the i64-th bit or any other bit on its left is set then
+      // it means that there are no more documents in this block, go to the
+      // next one
+      return firstDoc(i4096 + 1);
+    } else {
+      // We know we still have some 64-bits blocks that have bits set, let's
+      // advance to the next one by skipping trailing zeros of the index
+      int i1 = i & 0x3F;
+      int trailingZeros = Long.numberOfTrailingZeros(indexBits);
+      if (trailingZeros != 0) {
+        // no bits in the current long, go to the next one
+        i64 += trailingZeros;
+        i1 = 0;
+      }
 
-    @Override
-    public int advance(int target) throws IOException {
-      final int i4096 = target >>> 12;
-      if (i4096 >= indices.length) {
-        return doc = NO_MORE_DOCS;
+      // So now we are on a sub 64-bits block that has values
+      assert (index & (1L << i64)) != 0;
+      // we count the number of ones on the left of i64 to figure out the
+      // index of the long that contains the bits we are interested in
+      int longIndex = Long.bitCount(index & ((1L << i64) - 1)); // shifts are mod 64 in java
+      final long[] longArray = bits[i4096];
+      assert longArray[longIndex] != 0;
+      long bits = longArray[longIndex] >>> i1; // shifts are mod 64 in java
+      if (bits != 0L) {
+        // hurray, we found some non-zero bits, this gives us the next document:
+        i1 += Long.numberOfTrailingZeros(bits);
+        return (i4096 << 12) | ((i64 & 0x3F) << 6) | i1;
       }
-      final long index = indices[i4096];
-      int i64 = target >>> 6;
-      long indexBits = index >>> i64;
+
+      // otherwise it means that although we were on a sub-64 block that contains
+      // documents, all documents of this sub-block have already been consumed
+      // so two cases:
+      indexBits = index >>> i64 >>> 1; // we don't shift by (i64+1) otherwise we might shift by a multiple of 64 which is a no-op
       if (indexBits == 0) {
-        // if the index is zero, it means that there is no value in the
-        // current block, so return the first document of the next block
-        // or
-        // if neither the i64-th bit or any other bit on its left is set then
-        // it means that there are no more documents in this block, go to the
-        // next one
+        // Case 1: this was the last long of the block of 4096 bits, then go
+        // to the next block
         return firstDoc(i4096 + 1);
+      }
+      // Case 2: go to the next sub 64-bits block in the current block of 4096 bits
+      // by skipping trailing zeros of the index
+      trailingZeros = Long.numberOfTrailingZeros(indexBits);
+      i64 += 1 + trailingZeros;
+      bits = longArray[longIndex + 1];
+      assert bits != 0;
+      i1 = Long.numberOfTrailingZeros(bits);
+      return (i4096 << 12) | ((i64 & 0x3F) << 6) | i1;
+    }
+  }
+
+  /** Return the long bits at the given <code>i64</code> index. */
+  private long longBits(long index, long[] bits, int i64) {
+    if ((index & (1L << i64)) == 0) {
+      return 0L;
+    } else {
+      return bits[Long.bitCount(index & ((1L << i64) - 1))];
+    }
+  }
+
+  private void or(final int i4096, final long index, long[] bits, int nonZeroLongCount) {
+    assert Long.bitCount(index) == nonZeroLongCount;
+    final long currentIndex = indices[i4096];
+    if (currentIndex == 0) {
+      // fast path: if we currently have nothing in the block, just copy the data
+      // this especially happens all the time if you call OR on an empty set
+      indices[i4096] = index;
+      this.bits[i4096] = Arrays.copyOf(bits, nonZeroLongCount);
+      this.nonZeroLongCount += nonZeroLongCount;
+      return;
+    }
+    final long[] currentBits = this.bits[i4096];
+    final long[] newBits;
+    final long newIndex = currentIndex | index;
+    final int requiredCapacity = Long.bitCount(newIndex);
+    if (currentBits.length >= requiredCapacity) {
+      newBits = currentBits;
+    } else {
+      newBits = new long[oversize(requiredCapacity)];
+    }
+    // we iterate backwards in order to not override data we might need on the next iteration if the
+    // array is reused
+    for (int i = Long.numberOfLeadingZeros(newIndex), newO = Long.bitCount(newIndex) - 1;
+        i < 64;
+        i += 1 + Long.numberOfLeadingZeros(newIndex << (i + 1)), newO -= 1) {
+      // bitIndex is the index of a bit which is set in newIndex and newO is the number of 1 bits on its right
+      final int bitIndex = 63 - i;
+      assert newO == Long.bitCount(newIndex & ((1L << bitIndex) - 1));
+      newBits[newO] = longBits(currentIndex, currentBits, bitIndex) | longBits(index, bits, bitIndex);
+    }
+    indices[i4096] = newIndex;
+    this.bits[i4096] = newBits;
+    this.nonZeroLongCount += nonZeroLongCount - Long.bitCount(currentIndex & index);
+  }
+
+  private void or(SparseFixedBitSet other) {
+    for (int i = 0; i < other.indices.length; ++i) {
+      final long index = other.indices[i];
+      if (index != 0) {
+        or(i, index, other.bits[i], Long.bitCount(index));
+      }
+    }
+  }
+
+  /**
+   * {@link #or(DocIdSetIterator)} impl that works best when <code>it</code> is dense
+   */
+  private void orDense(DocIdSetIterator it) throws IOException {
+    assertUnpositioned(it);
+    // The goal here is to try to take advantage of the ordering of documents
+    // to build the data-structure more efficiently
+    // NOTE: this heavily relies on the fact that shifts are mod 64
+    final int firstDoc = it.nextDoc();
+    if (firstDoc == DocIdSetIterator.NO_MORE_DOCS) {
+      return;
+    }
+    int i4096 = firstDoc >>> 12;
+    int i64 = firstDoc >>> 6;
+    long index = 1L << i64;
+    long currentLong = 1L << firstDoc;
+    // we store at most 64 longs per block so preallocate in order never to have to resize
+    long[] longs = new long[64];
+    int numLongs = 0;
+
+    for (int doc = it.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = it.nextDoc()) {
+      final int doc64 = doc >>> 6;
+      if (doc64 == i64) {
+        // still in the same long, just set the bit
+        currentLong |= 1L << doc;
       } else {
-        // We know we still have some 64-bits blocks that have bits set, let's
-        // advance to the next one by skipping trailing zeros of the index
-        int i1 = target & 0x3F;
-        int trailingZeros = Long.numberOfTrailingZeros(indexBits);
-        if (trailingZeros != 0) {
-          // no bits in the current long, go to the next one
-          i64 += trailingZeros;
-          i1 = 0;
-        }
+        longs[numLongs++] = currentLong;
 
-        // So now we are on a sub 64-bits block that has values
-        assert (index & (1L << i64)) != 0;
-        // we count the number of ones on the left of i64 to figure out the
-        // index of the long that contains the bits we are interested in
-        int longIndex = Long.bitCount(index & ((1L << i64) - 1)); // shifts are mod 64 in java
-        final long[] longArray = bits[i4096];
-        assert longArray[longIndex] != 0;
-        long bits = longArray[longIndex] >>> i1; // shifts are mod 64 in java
-        if (bits != 0L) {
-          // hurray, we found some non-zero bits, this gives us the next document:
-          i1 += Long.numberOfTrailingZeros(bits);
-          return doc = (i4096 << 12) | ((i64 & 0x3F) << 6) | i1;
+        final int doc4096 = doc >>> 12;
+        if (doc4096 == i4096) {
+          index |= 1L << doc64;
+        } else {
+          // we are on a new block, flush what we buffered
+          or(i4096, index, longs, numLongs);
+          // and reset state for the new block
+          i4096 = doc4096;
+          index = 1L << doc64;
+          numLongs = 0;
         }
 
-        // otherwise it means that although we were on a sub-64 block that contains
-        // documents, all documents of this sub-block have already been consumed
-        // so two cases:
-        indexBits = index >>> i64 >>> 1; // we don't shift by (i64+1) otherwise we might shift by a multiple of 64 which is a no-op
-        if (indexBits == 0) {
-          // Case 1: this was the last long of the block of 4096 bits, then go
-          // to the next block
-          return firstDoc(i4096 + 1);
-        }
-        // Case 2: go to the next sub 64-bits block in the current block of 4096 bits
-        // by skipping trailing zeros of the index
-        trailingZeros = Long.numberOfTrailingZeros(indexBits);
-        i64 += 1 + trailingZeros;
-        bits = longArray[longIndex + 1];
-        assert bits != 0;
-        i1 = Long.numberOfTrailingZeros(bits);
-        return doc = (i4096 << 12) | ((i64 & 0x3F) << 6) | i1;
+        // we are on a new long, reset state
+        i64 = doc64;
+        currentLong = 1L << doc;
       }
     }
 
-    @Override
-    public long cost() {
-      // although constant-time, approximateCardinality is a bit expensive so
-      // we cache it to avoid performance traps eg. when sorting iterators by
-      // cost
-      if (cost < 0) {
-        cost = approximateCardinality();
+    // flush
+    longs[numLongs++] = currentLong;
+    or(i4096, index, longs, numLongs);
+  }
+
+  @Override
+  public void or(DocIdSetIterator it) throws IOException {
+    {
+      // specialize union with another SparseFixedBitSet
+      final SparseFixedBitSet other = BitSetIterator.getSparseFixedBitSetOrNull(it);
+      if (other != null) {
+        assertUnpositioned(it);
+        or(other);
+        return;
       }
-      assert cost >= 0;
-      return cost;
     }
 
+    // We do not specialize the union with a FixedBitSet since FixedBitSets are
+    // supposed to be used for dense data and sparse fixed bit sets for sparse
+    // data, so a sparse set would likely get upgraded by DocIdSetBuilder before
+    // being or'ed with a FixedBitSet
+
+    if (it.cost() < indices.length) {
+      // the default impl is good for sparse iterators
+      super.or(it);
+    } else {
+      orDense(it);
+    }
+  }
+
+  // AND and AND_NOT do not need much specialization here since this sparse set
+  // is supposed to be used on sparse data and the default AND/AND_NOT impl
+  // (leap frog) is efficient when at least one of the sets contains sparse data
+
+  @Override
+  public void and(DocIdSetIterator it) throws IOException {
+    final SparseFixedBitSet other = BitSetIterator.getSparseFixedBitSetOrNull(it);
+    if (other != null) {
+      // if we are merging with another SparseFixedBitSet, a quick win is
+      // to clear up some blocks by only looking at their index. Then the set
+      // is sparser and the leap-frog approach of the parent class is more
+      // efficient. Since SparseFixedBitSet is supposed to be used for sparse
+      // sets, the intersection of two SparseFixedBitSet is likely very sparse
+      final int numCommonBlocks = Math.min(indices.length, other.indices.length);
+      for (int i = 0; i < numCommonBlocks; ++i) {
+        if ((indices[i] & other.indices[i]) == 0) {
+          this.nonZeroLongCount -= Long.bitCount(this.indices[i]);
+          this.indices[i] = 0;
+          this.bits[i] = null;
+        }
+      }
+    }
+    super.and(it);
+  }
+
+  @Override
+  public long ramBytesUsed() {
+    return ramBytesUsed;
   }
 
 }

Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/Version.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/Version.java?rev=1634823&r1=1634822&r2=1634823&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/Version.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/Version.java Tue Oct 28 09:19:21 2014
@@ -33,153 +33,6 @@ import java.util.Locale;
 public final class Version {
 
   /**
-   * Match settings and bugs in Lucene's 4.0.0-ALPHA release.
-   * @deprecated (5.0) Use latest
-   */
-  @Deprecated
-  public static final Version LUCENE_4_0_0_ALPHA = new Version(4, 0, 0, 0);
-
-  /**
-   * Match settings and bugs in Lucene's 4.0.0-BETA release.
-   * @deprecated (5.0) Use latest
-   */
-  @Deprecated
-  public static final Version LUCENE_4_0_0_BETA = new Version(4, 0, 0, 1);
-
-  /**
-   * Match settings and bugs in Lucene's 4.0.0 release.
-   * @deprecated (5.0) Use latest
-   */
-  @Deprecated
-  public static final Version LUCENE_4_0_0 = new Version(4, 0, 0, 2);
-  
-  /**
-   * Match settings and bugs in Lucene's 4.1.0 release.
-   * @deprecated (5.0) Use latest
-   */
-  @Deprecated
-  public static final Version LUCENE_4_1_0 = new Version(4, 1, 0);
-
-  /**
-   * Match settings and bugs in Lucene's 4.2.0 release.
-   * @deprecated (5.0) Use latest
-   */
-  @Deprecated
-  public static final Version LUCENE_4_2_0 = new Version(4, 2, 0);
-
-  /**
-   * Match settings and bugs in Lucene's 4.2.1 release.
-   * @deprecated (5.0) Use latest
-   */
-  @Deprecated
-  public static final Version LUCENE_4_2_1 = new Version(4, 2, 1);
-
-  /**
-   * Match settings and bugs in Lucene's 4.3.0 release.
-   * @deprecated (5.0) Use latest
-   */
-  @Deprecated
-  public static final Version LUCENE_4_3_0 = new Version(4, 3, 0);
-
-  /**
-   * Match settings and bugs in Lucene's 4.3.1 release.
-   * @deprecated (5.0) Use latest
-   */
-  @Deprecated
-  public static final Version LUCENE_4_3_1 = new Version(4, 3, 1);
-
-  /**
-   * Match settings and bugs in Lucene's 4.4.0 release.
-   * @deprecated (5.0) Use latest
-   */
-  @Deprecated
-  public static final Version LUCENE_4_4_0 = new Version(4, 4, 0);
-
-  /**
-   * Match settings and bugs in Lucene's 4.5.0 release.
-   * @deprecated (5.0) Use latest
-   */
-  @Deprecated
-  public static final Version LUCENE_4_5_0 = new Version(4, 5, 0);
-
-  /**
-   * Match settings and bugs in Lucene's 4.5.1 release.
-   * @deprecated (5.0) Use latest
-   */
-  @Deprecated
-  public static final Version LUCENE_4_5_1 = new Version(4, 5, 1);
-
-  /**
-   * Match settings and bugs in Lucene's 4.6.0 release.
-   * @deprecated (5.0) Use latest
-   */
-  @Deprecated
-  public static final Version LUCENE_4_6_0 = new Version(4, 6, 0);
-
-  /**
-   * Match settings and bugs in Lucene's 4.6.1 release.
-   * @deprecated (5.0) Use latest
-   */
-  @Deprecated
-  public static final Version LUCENE_4_6_1 = new Version(4, 6, 1);
-  
-  /**
-   * Match settings and bugs in Lucene's 4.7.0 release.
-   * @deprecated (5.0) Use latest
-   */
-  @Deprecated
-  public static final Version LUCENE_4_7_0 = new Version(4, 7, 0);
-
-  /**
-   * Match settings and bugs in Lucene's 4.7.1 release.
-   * @deprecated (5.0) Use latest
-   */
-  @Deprecated
-  public static final Version LUCENE_4_7_1 = new Version(4, 7, 1);
-
-  /**
-   * Match settings and bugs in Lucene's 4.7.2 release.
-   * @deprecated (5.0) Use latest
-   */
-  @Deprecated
-  public static final Version LUCENE_4_7_2 = new Version(4, 7, 2);
-  
-  /**
-   * Match settings and bugs in Lucene's 4.8.0 release.
-   * @deprecated (5.0) Use latest
-   */
-  @Deprecated
-  public static final Version LUCENE_4_8_0 = new Version(4, 8, 0);
-
-  /**
-   * Match settings and bugs in Lucene's 4.8.1 release.
-   * @deprecated (5.0) Use latest
-   */
-  @Deprecated
-  public static final Version LUCENE_4_8_1 = new Version(4, 8, 1);
-
-  /**
-   * Match settings and bugs in Lucene's 4.9.0 release.
-   * @deprecated (5.0) Use latest
-   */
-  @Deprecated
-  public static final Version LUCENE_4_9_0 = new Version(4, 9, 0);
-  
-  /**
-   * Match settings and bugs in Lucene's 4.10.0 release.
-   * @deprecated (5.0) Use latest
-   */
-  @Deprecated
-  public static final Version LUCENE_4_10_0 = new Version(4, 10, 0);
-
-  /**
-   * Match settings and bugs in Lucene's 4.10.1 release.
-   * @deprecated (5.0) Use latest
-   */
-  @Deprecated
-  public static final Version LUCENE_4_10_1 = new Version(4, 10, 1);
-
-  /**
    * Match settings and bugs in Lucene's 5.0 release.
    * @deprecated (5.0) Use latest
    */
@@ -221,47 +74,6 @@ public final class Version {
   @Deprecated
   public static final Version LUCENE_CURRENT = LATEST;
 
-  /** @deprecated Bad naming of constant; use {@link #LUCENE_4_0_0} instead (this constant actually points to {@link #LUCENE_4_0_0_ALPHA} to match whole 4.0 series). */
-  @Deprecated
-  public static final Version LUCENE_4_0 = LUCENE_4_0_0_ALPHA;
-
-  /** @deprecated Bad naming of constant; use {@link #LUCENE_4_1_0} instead. */
-  @Deprecated
-  public static final Version LUCENE_4_1 = LUCENE_4_1_0;
-
-  /** @deprecated Bad naming of constant; use {@link #LUCENE_4_2_0} instead. */
-  @Deprecated
-  public static final Version LUCENE_4_2 = LUCENE_4_2_0;
-
-  /** @deprecated Bad naming of constant; use {@link #LUCENE_4_3_0} instead. */
-  @Deprecated
-  public static final Version LUCENE_4_3 = LUCENE_4_3_0;
-
-  /** @deprecated Bad naming of constant; use {@link #LUCENE_4_4_0} instead. */
-  @Deprecated
-  public static final Version LUCENE_4_4 = LUCENE_4_4_0;
-
-  /** @deprecated Bad naming of constant; use {@link #LUCENE_4_5_0} instead. */
-  @Deprecated
-  public static final Version LUCENE_4_5 = LUCENE_4_5_0;
-
-  /** @deprecated Bad naming of constant; use {@link #LUCENE_4_6_0} instead. */
-  @Deprecated
-  public static final Version LUCENE_4_6 = LUCENE_4_6_0;
-
-  /** @deprecated Bad naming of constant; use {@link #LUCENE_4_7_0} instead. */
-  @Deprecated
-  public static final Version LUCENE_4_7 = LUCENE_4_7_0;
-
-  /** @deprecated Bad naming of constant; use {@link #LUCENE_4_8_0} instead. */
-  @Deprecated
-  public static final Version LUCENE_4_8 = LUCENE_4_8_0;
-
-  /** @deprecated Bad naming of constant; use {@link #LUCENE_4_9_0} instead. */
-  @Deprecated
-  public static final Version LUCENE_4_9 = LUCENE_4_9_0;
-
-
   /**
    * Parse a version number of the form {@code "major.minor.bugfix.prerelease"}.
    *
@@ -350,12 +162,6 @@ public final class Version {
       case "LATEST":
       case "LUCENE_CURRENT":
         return LATEST;
-      case "LUCENE_4_0_0":
-        return LUCENE_4_0_0;
-      case "LUCENE_4_0_0_ALPHA":
-        return LUCENE_4_0_0_ALPHA;
-      case "LUCENE_4_0_0_BETA":
-        return LUCENE_4_0_0_BETA;
       default:
         version = version
           .replaceFirst("^LUCENE_(\\d+)_(\\d+)_(\\d+)$", "$1.$2.$3")

Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/packed/AbstractPagedMutable.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/packed/AbstractPagedMutable.java?rev=1634823&r1=1634822&r2=1634823&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/packed/AbstractPagedMutable.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/packed/AbstractPagedMutable.java Tue Oct 28 09:19:21 2014
@@ -20,6 +20,9 @@ package org.apache.lucene.util.packed;
 import static org.apache.lucene.util.packed.PackedInts.checkBlockSize;
 import static org.apache.lucene.util.packed.PackedInts.numBlocks;
 
+import java.util.Collections;
+
+import org.apache.lucene.util.Accountable;
 import org.apache.lucene.util.ArrayUtil;
 import org.apache.lucene.util.LongValues;
 import org.apache.lucene.util.RamUsageEstimator;
@@ -28,7 +31,7 @@ import org.apache.lucene.util.RamUsageEs
  * Base implementation for {@link PagedMutable} and {@link PagedGrowableWriter}.
  * @lucene.internal
  */
-abstract class AbstractPagedMutable<T extends AbstractPagedMutable<T>> extends LongValues {
+abstract class AbstractPagedMutable<T extends AbstractPagedMutable<T>> extends LongValues implements Accountable {
 
   static final int MIN_BLOCK_SIZE = 1 << 6;
   static final int MAX_BLOCK_SIZE = 1 << 30;
@@ -104,7 +107,7 @@ abstract class AbstractPagedMutable<T ex
         + 3 * RamUsageEstimator.NUM_BYTES_INT;
   }
 
-  /** Return the number of bytes used by this object. */
+  @Override
   public long ramBytesUsed() {
     long bytesUsed = RamUsageEstimator.alignObjectSize(baseRamBytesUsed());
     bytesUsed += RamUsageEstimator.alignObjectSize(RamUsageEstimator.shallowSizeOf(subMutables));
@@ -113,6 +116,11 @@ abstract class AbstractPagedMutable<T ex
     }
     return bytesUsed;
   }
+  
+  @Override
+  public Iterable<? extends Accountable> getChildResources() {
+    return Collections.emptyList();
+  }
 
   protected abstract T newUnfilledCopy(long newSize);
 

Modified: lucene/dev/branches/lucene6005/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.DocValuesFormat
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.DocValuesFormat?rev=1634823&r1=1634822&r2=1634823&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.DocValuesFormat (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.DocValuesFormat Tue Oct 28 09:19:21 2014
@@ -13,4 +13,4 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
-org.apache.lucene.codecs.lucene410.Lucene410DocValuesFormat
+org.apache.lucene.codecs.lucene50.Lucene50DocValuesFormat

Modified: lucene/dev/branches/lucene6005/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat?rev=1634823&r1=1634822&r2=1634823&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat Tue Oct 28 09:19:21 2014
@@ -13,4 +13,4 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
-org.apache.lucene.codecs.lucene41.Lucene41PostingsFormat
+org.apache.lucene.codecs.lucene50.Lucene50PostingsFormat

Modified: lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/TestAssertions.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/TestAssertions.java?rev=1634823&r1=1634822&r2=1634823&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/TestAssertions.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/TestAssertions.java Tue Oct 28 09:19:21 2014
@@ -17,8 +17,8 @@ package org.apache.lucene;
  * limitations under the License.
  */
 
-import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.util.LuceneTestCase;
 
 /**
  * validate that assertions are enabled during tests
@@ -43,13 +43,13 @@ public class TestAssertions extends Luce
   public void testTokenStreams() {
     new TestTokenStream1();
     new TestTokenStream2();
-    boolean doFail = false;
     try {
       new TestTokenStream3();
-      doFail = true;
+      if (assertsAreEnabled) {
+        fail("TestTokenStream3 should fail assertion");
+      }
     } catch (AssertionError e) {
       // expected
     }
-    assertFalse("TestTokenStream3 should fail assertion", doFail);
   }
 }

Modified: lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/codecs/lucene50/TestLucene50NormsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/codecs/lucene50/TestLucene50NormsFormat.java?rev=1634823&r1=1634822&r2=1634823&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/codecs/lucene50/TestLucene50NormsFormat.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/codecs/lucene50/TestLucene50NormsFormat.java Tue Oct 28 09:19:21 2014
@@ -17,7 +17,13 @@ package org.apache.lucene.codecs.lucene5
  * limitations under the License.
  */
 
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
 import org.apache.lucene.codecs.Codec;
+import org.apache.lucene.codecs.lucene50.Lucene50NormsConsumer.NormMap;
 import org.apache.lucene.index.BaseNormsFormatTestCase;
 import org.apache.lucene.util.TestUtil;
 
@@ -30,5 +36,92 @@ public class TestLucene50NormsFormat ext
   @Override
   protected Codec getCodec() {
     return codec;
-  } 
+  }
+  
+  // NormMap is rather complicated, doing domain encoding / tracking frequencies etc.
+  // test it directly some here...
+
+  public void testNormMapSimple() {
+    NormMap map = new NormMap();
+    map.add(10);
+    map.add(5);
+    map.add(4);
+    map.add(10);
+    assertEquals(3, map.size);
+    
+    // first come, first serve ord assignment
+    
+    // encode
+    assertEquals(0, map.getOrd(10));
+    assertEquals(1, map.getOrd(5));
+    assertEquals(2, map.getOrd(4));
+    
+    // decode
+    long decode[] = map.getDecodeTable();
+    assertEquals(10, decode[0]);
+    assertEquals(5, decode[1]);
+    assertEquals(4, decode[2]);
+    
+    // freqs
+    int freqs[] = map.getFreqs();
+    assertEquals(2, freqs[0]);
+    assertEquals(1, freqs[1]);
+    assertEquals(1, freqs[2]);
+    
+    assertEquals(2, map.maxFreq());
+  }
+  
+  public void testNormMapRandom() {
+    Map<Long,Integer> freqs = new HashMap<>();
+    Map<Long,Integer> ords = new HashMap<>();
+    
+    Set<Long> uniqueValuesSet = new HashSet<>();
+    int numUniqValues = TestUtil.nextInt(random(), 1, 256);
+    for (int i = 0; i < numUniqValues; i++) {
+      if (random().nextBoolean()) {
+        uniqueValuesSet.add(TestUtil.nextLong(random(), Long.MIN_VALUE, Long.MAX_VALUE));
+      } else {
+        uniqueValuesSet.add(TestUtil.nextLong(random(), Byte.MIN_VALUE, Byte.MAX_VALUE));
+      }
+    }
+    
+    Long uniqueValues[] = uniqueValuesSet.toArray(new Long[uniqueValuesSet.size()]);
+    
+    NormMap map = new NormMap();
+    int numdocs = TestUtil.nextInt(random(), 1, 100000);
+    for (int i = 0; i < numdocs; i++) {
+      long value = uniqueValues[random().nextInt(uniqueValues.length)];
+      // now add to both expected and actual
+      map.add(value);
+      
+      Integer ord = ords.get(value);
+      if (ord == null) {
+        ord = ords.size();
+        ords.put(value, ord);
+        freqs.put(value, 1);
+      } else {
+        freqs.put(value, freqs.get(value)+1);
+      }
+    }
+    
+    // value -> ord
+    assertEquals(ords.size(), map.size);
+    for (Map.Entry<Long,Integer> kv : ords.entrySet()) {
+      assertEquals(kv.getValue().intValue(), map.getOrd(kv.getKey()));
+    }
+    
+    // ord -> value
+    Map<Long,Integer> reversed = new HashMap<>();
+    long table[] = map.getDecodeTable();
+    for (int i = 0; i < map.size; i++) {
+      reversed.put(table[i], i);
+    }
+    assertEquals(ords, reversed);
+    
+    // freqs
+    int freqTable[] = map.getFreqs();
+    for (int i = 0; i < map.size; i++) {
+      assertEquals(freqs.get(table[i]).longValue(), freqTable[i]);
+    }
+  }
 }

Modified: lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/codecs/perfield/TestPerFieldPostingsFormat2.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/codecs/perfield/TestPerFieldPostingsFormat2.java?rev=1634823&r1=1634822&r2=1634823&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/codecs/perfield/TestPerFieldPostingsFormat2.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/codecs/perfield/TestPerFieldPostingsFormat2.java Tue Oct 28 09:19:21 2014
@@ -22,7 +22,7 @@ import org.apache.lucene.analysis.MockAn
 import org.apache.lucene.codecs.Codec;
 import org.apache.lucene.codecs.PostingsFormat;
 import org.apache.lucene.codecs.asserting.AssertingCodec;
-import org.apache.lucene.codecs.lucene41vargap.Lucene41VarGapFixedInterval;
+import org.apache.lucene.codecs.blockterms.LuceneVarGapFixedInterval;
 import org.apache.lucene.codecs.memory.MemoryPostingsFormat;
 import org.apache.lucene.codecs.simpletext.SimpleTextPostingsFormat;
 import org.apache.lucene.document.Document;
@@ -287,9 +287,9 @@ public class TestPerFieldPostingsFormat2
       @Override
       public PostingsFormat getPostingsFormatForField(String field) {
         if ("id".equals(field)) {
-          return new Lucene41VarGapFixedInterval(1);
+          return new LuceneVarGapFixedInterval(1);
         } else if ("date".equals(field)) {
-          return new Lucene41VarGapFixedInterval(2);
+          return new LuceneVarGapFixedInterval(2);
         } else {
           return super.getPostingsFormatForField(field);
         }

Modified: lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/document/TestDocument2.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/document/TestDocument2.java?rev=1634823&r1=1634822&r2=1634823&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/document/TestDocument2.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/document/TestDocument2.java Tue Oct 28 09:19:21 2014
@@ -706,7 +706,7 @@ public class TestDocument2 extends Lucen
     assertEquals(1, checked.segmentInfos.size());
     CheckIndex.Status.SegmentInfoStatus segment = checked.segmentInfos.get(0);
     assertNotNull(segment.termIndexStatus.blockTreeStats);
-    Stats btStats = segment.termIndexStatus.blockTreeStats.get("id");
+    Stats btStats = (Stats) segment.termIndexStatus.blockTreeStats.get("id");
     assertNotNull(btStats);
     assertEquals(2, btStats.termsOnlyBlockCount);
     assertEquals(1, btStats.subBlocksOnlyBlockCount);
@@ -725,7 +725,7 @@ public class TestDocument2 extends Lucen
       fail("did not hit exception");
     } catch (IllegalArgumentException iae) {
       // Expected
-      assertTrue("wrong exception message: " + iae.getMessage(), iae.getMessage().startsWith("field \"id\": A SPI class of type org.apache.lucene.codecs.DocValuesFormat with name 'foobar' does not exist"));
+      assertTrue("wrong exception message: " + iae.getMessage(), iae.getMessage().startsWith("field \"id\": An SPI class of type org.apache.lucene.codecs.DocValuesFormat with name 'foobar' does not exist"));
     }
     fieldTypes.setDocValuesFormat("id", "Memory");
     w.close();
@@ -761,7 +761,7 @@ public class TestDocument2 extends Lucen
       fail("did not hit exception");
     } catch (IllegalArgumentException iae) {
       // Expected
-      assertTrue("wrong exception message: " + iae.getMessage(), iae.getMessage().startsWith("field \"id\": A SPI class of type org.apache.lucene.codecs.PostingsFormat with name 'foobar' does not exist"));
+      assertTrue("wrong exception message: " + iae.getMessage(), iae.getMessage().startsWith("field \"id\": An SPI class of type org.apache.lucene.codecs.PostingsFormat with name 'foobar' does not exist"));
     }
     fieldTypes.setPostingsFormat("id", "Memory");
     w.close();

Modified: lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestAddIndexes.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestAddIndexes.java?rev=1634823&r1=1634822&r2=1634823&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestAddIndexes.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestAddIndexes.java Tue Oct 28 09:19:21 2014
@@ -1116,8 +1116,7 @@ public class TestAddIndexes extends Luce
     w3.close();
     // we should now see segments_X,
     // _Y.cfs,_Y.cfe, _Z.si
-    SegmentInfos sis = new SegmentInfos();
-    sis.read(dir);
+    SegmentInfos sis = SegmentInfos.readLatestCommit(dir);
     assertEquals("Only one compound segment should exist", 1, sis.size());
     assertTrue(sis.info(0).info.getUseCompoundFile());
     dir.close();

Modified: lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestAllFilesHaveChecksumFooter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestAllFilesHaveChecksumFooter.java?rev=1634823&r1=1634822&r2=1634823&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestAllFilesHaveChecksumFooter.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestAllFilesHaveChecksumFooter.java Tue Oct 28 09:19:21 2014
@@ -26,7 +26,6 @@ import org.apache.lucene.document.Field;
 import org.apache.lucene.document.NumericDocValuesField;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.IndexInput;
-import org.apache.lucene.store.MockDirectoryWrapper;
 import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.util.TestUtil;
 
@@ -36,10 +35,6 @@ import org.apache.lucene.util.TestUtil;
 public class TestAllFilesHaveChecksumFooter extends LuceneTestCase {
   public void test() throws Exception {
     Directory dir = newDirectory();
-    if (dir instanceof MockDirectoryWrapper) {
-      // Else we might remove .cfe but not the corresponding .cfs, causing false exc when trying to verify headers:
-      ((MockDirectoryWrapper) dir).setEnableVirusScanner(false);
-    }
     IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
     conf.setCodec(TestUtil.getDefaultCodec());
     RandomIndexWriter riw = new RandomIndexWriter(random(), dir, conf);
@@ -68,8 +63,7 @@ public class TestAllFilesHaveChecksumFoo
   }
   
   private void checkFooters(Directory dir) throws IOException {
-    SegmentInfos sis = new SegmentInfos();
-    sis.read(dir);
+    SegmentInfos sis = SegmentInfos.readLatestCommit(dir);
     checkFooter(dir, sis.getSegmentsFileName());
     
     for (SegmentCommitInfo si : sis) {

Modified: lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestAllFilesHaveCodecHeader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestAllFilesHaveCodecHeader.java?rev=1634823&r1=1634822&r2=1634823&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestAllFilesHaveCodecHeader.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestAllFilesHaveCodecHeader.java Tue Oct 28 09:19:21 2014
@@ -30,22 +30,16 @@ import org.apache.lucene.document.Numeri
 import org.apache.lucene.document.TextField;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.IndexInput;
-import org.apache.lucene.store.MockDirectoryWrapper;
 import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.util.TestUtil;
 
 /**
- * Test that a plain default puts codec headers in all files.
+ * Test that a plain default puts codec headers in all files
  */
 public class TestAllFilesHaveCodecHeader extends LuceneTestCase {
   public void test() throws Exception {
     Directory dir = newDirectory();
 
-    if (dir instanceof MockDirectoryWrapper) {
-      // Else we might remove .cfe but not the corresponding .cfs, causing false exc when trying to verify headers:
-      ((MockDirectoryWrapper) dir).setEnableVirusScanner(false);
-    }
-
     IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
     conf.setCodec(TestUtil.getDefaultCodec());
     RandomIndexWriter riw = new RandomIndexWriter(random(), dir, conf);
@@ -70,10 +64,12 @@ public class TestAllFilesHaveCodecHeader
       if (random().nextInt(7) == 0) {
         riw.commit();
       }
-      // TODO: we should make a new format with a clean header...
-      // if (random().nextInt(20) == 0) {
-      //  riw.deleteDocuments(new Term("id", Integer.toString(i)));
-      // }
+      if (random().nextInt(20) == 0) {
+        riw.deleteDocuments(new Term("id", Integer.toString(i)));
+      }
+      if (random().nextInt(15) == 0) {
+        riw.updateNumericDocValue(new Term("id"), "dv", Long.valueOf(i));
+      }
     }
     riw.close();
     checkHeaders(dir, new HashMap<String,String>());
@@ -81,25 +77,25 @@ public class TestAllFilesHaveCodecHeader
   }
   
   private void checkHeaders(Directory dir, Map<String,String> namesToExtensions) throws IOException {
-    SegmentInfos sis = new SegmentInfos();
-    sis.read(dir);
-    checkHeader(dir, sis.getSegmentsFileName(), namesToExtensions);
+    SegmentInfos sis = SegmentInfos.readLatestCommit(dir);
+    checkHeader(dir, sis.getSegmentsFileName(), namesToExtensions, sis.getId());
     
     for (SegmentCommitInfo si : sis) {
+      assertNotNull(si.info.getId());
       for (String file : si.files()) {
-        checkHeader(dir, file, namesToExtensions);
+        checkHeader(dir, file, namesToExtensions, si.info.getId());
       }
       if (si.info.getUseCompoundFile()) {
         try (Directory cfsDir = si.info.getCodec().compoundFormat().getCompoundReader(dir, si.info, newIOContext(random()))) {
           for (String cfsFile : cfsDir.listAll()) {
-            checkHeader(cfsDir, cfsFile, namesToExtensions);
+            checkHeader(cfsDir, cfsFile, namesToExtensions, si.info.getId());
           }
         }
       }
     }
   }
   
-  private void checkHeader(Directory dir, String file, Map<String,String> namesToExtensions) throws IOException {
+  private void checkHeader(Directory dir, String file, Map<String,String> namesToExtensions, byte[] id) throws IOException {
     try (IndexInput in = dir.openInput(file, newIOContext(random()))) {
       int val = in.readInt();
       assertEquals(file + " has no codec header, instead found: " + val, CodecUtil.CODEC_MAGIC, val);
@@ -114,6 +110,10 @@ public class TestAllFilesHaveCodecHeader
       if (previous != null && !previous.equals(extension)) {
         fail("extensions " + previous + " and " + extension + " share same codecName " + codecName);
       }
+      // read version
+      in.readInt();
+      // read object id
+      CodecUtil.checkIndexHeaderID(in, id);      
     }
   }
 }

Modified: lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestCodecUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestCodecUtil.java?rev=1634823&r1=1634822&r2=1634823&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestCodecUtil.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestCodecUtil.java Tue Oct 28 09:19:21 2014
@@ -199,12 +199,12 @@ public class TestCodecUtil extends Lucen
   public void testSegmentHeaderLength() throws Exception {
     RAMFile file = new RAMFile();
     IndexOutput output = new RAMOutputStream(file, true);
-    CodecUtil.writeSegmentHeader(output, "FooBar", 5, StringHelper.randomId(), "xyz");
+    CodecUtil.writeIndexHeader(output, "FooBar", 5, StringHelper.randomId(), "xyz");
     output.writeString("this is the data");
     output.close();
     
     IndexInput input = new RAMInputStream("file", file);
-    input.seek(CodecUtil.segmentHeaderLength("FooBar", "xyz"));
+    input.seek(CodecUtil.indexHeaderLength("FooBar", "xyz"));
     assertEquals("this is the data", input.readString());
     input.close();
   }
@@ -217,7 +217,7 @@ public class TestCodecUtil extends Lucen
     RAMFile file = new RAMFile();
     IndexOutput output = new RAMOutputStream(file, true);
     try {
-      CodecUtil.writeSegmentHeader(output, "foobar", 5, StringHelper.randomId(), tooLong.toString());
+      CodecUtil.writeIndexHeader(output, "foobar", 5, StringHelper.randomId(), tooLong.toString());
       fail("didn't get expected exception");
     } catch (IllegalArgumentException expected) {
       // expected
@@ -232,13 +232,13 @@ public class TestCodecUtil extends Lucen
     RAMFile file = new RAMFile();
     IndexOutput output = new RAMOutputStream(file, true);
     byte[] id = StringHelper.randomId();
-    CodecUtil.writeSegmentHeader(output, "foobar", 5, id, justLongEnough.toString());
+    CodecUtil.writeIndexHeader(output, "foobar", 5, id, justLongEnough.toString());
     output.close();
     
     IndexInput input = new RAMInputStream("file", file);
-    CodecUtil.checkSegmentHeader(input, "foobar", 5, 5, id, justLongEnough.toString());
+    CodecUtil.checkIndexHeader(input, "foobar", 5, 5, id, justLongEnough.toString());
     assertEquals(input.getFilePointer(), input.length());
-    assertEquals(input.getFilePointer(), CodecUtil.segmentHeaderLength("foobar", justLongEnough.toString()));
+    assertEquals(input.getFilePointer(), CodecUtil.indexHeaderLength("foobar", justLongEnough.toString()));
     input.close();
   }
   
@@ -246,7 +246,7 @@ public class TestCodecUtil extends Lucen
     RAMFile file = new RAMFile();
     IndexOutput output = new RAMOutputStream(file, true);
     try {
-      CodecUtil.writeSegmentHeader(output, "foobar", 5, StringHelper.randomId(), "\u1234");
+      CodecUtil.writeIndexHeader(output, "foobar", 5, StringHelper.randomId(), "\u1234");
       fail("didn't get expected exception");
     } catch (IllegalArgumentException expected) {
       // expected

Modified: lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestConsistentFieldNumbers.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestConsistentFieldNumbers.java?rev=1634823&r1=1634822&r2=1634823&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestConsistentFieldNumbers.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestConsistentFieldNumbers.java Tue Oct 28 09:19:21 2014
@@ -64,8 +64,7 @@ public class TestConsistentFieldNumbers 
 
       writer.close();
 
-      SegmentInfos sis = new SegmentInfos();
-      sis.read(dir);
+      SegmentInfos sis = SegmentInfos.readLatestCommit(dir);
       assertEquals(2, sis.size());
 
       FieldInfos fis1 = IndexWriter.readFieldInfos(sis.info(0));
@@ -82,8 +81,7 @@ public class TestConsistentFieldNumbers 
       writer.forceMerge(1);
       writer.close();
 
-      sis = new SegmentInfos();
-      sis.read(dir);
+      sis = SegmentInfos.readLatestCommit(dir);
       assertEquals(1, sis.size());
 
       FieldInfos fis3 = IndexWriter.readFieldInfos(sis.info(0));
@@ -130,8 +128,7 @@ public class TestConsistentFieldNumbers 
     writer.addIndexes(dir2);
     writer.close();
 
-    SegmentInfos sis = new SegmentInfos();
-    sis.read(dir1);
+    SegmentInfos sis = SegmentInfos.readLatestCommit(dir1);
     assertEquals(2, sis.size());
 
     FieldInfos fis1 = IndexWriter.readFieldInfos(sis.info(0));
@@ -161,8 +158,7 @@ public class TestConsistentFieldNumbers 
         d.add(new TextField("f2", "d1 second field", Field.Store.YES));
         writer.addDocument(d);
         writer.close();
-        SegmentInfos sis = new SegmentInfos();
-        sis.read(dir);
+        SegmentInfos sis = SegmentInfos.readLatestCommit(dir);
         assertEquals(1, sis.size());
         FieldInfos fis1 = IndexWriter.readFieldInfos(sis.info(0));
         assertEquals("f1", fis1.fieldInfo(0).name);
@@ -178,8 +174,7 @@ public class TestConsistentFieldNumbers 
         d.add(new StoredField("f3", new byte[] { 1, 2, 3 }));
         writer.addDocument(d);
         writer.close();
-        SegmentInfos sis = new SegmentInfos();
-        sis.read(dir);
+        SegmentInfos sis = SegmentInfos.readLatestCommit(dir);
         assertEquals(2, sis.size());
         FieldInfos fis1 = IndexWriter.readFieldInfos(sis.info(0));
         FieldInfos fis2 = IndexWriter.readFieldInfos(sis.info(1));
@@ -199,8 +194,7 @@ public class TestConsistentFieldNumbers 
         d.add(new StoredField("f3", new byte[] { 1, 2, 3, 4, 5 }));
         writer.addDocument(d);
         writer.close();
-        SegmentInfos sis = new SegmentInfos();
-        sis.read(dir);
+        SegmentInfos sis = SegmentInfos.readLatestCommit(dir);
         assertEquals(3, sis.size());
         FieldInfos fis1 = IndexWriter.readFieldInfos(sis.info(0));
         FieldInfos fis2 = IndexWriter.readFieldInfos(sis.info(1));
@@ -231,8 +225,7 @@ public class TestConsistentFieldNumbers 
       writer.forceMerge(1);
       writer.close();
 
-      SegmentInfos sis = new SegmentInfos();
-      sis.read(dir);
+      SegmentInfos sis = SegmentInfos.readLatestCommit(dir);
       assertEquals(1, sis.size());
       FieldInfos fis1 = IndexWriter.readFieldInfos(sis.info(0));
       assertEquals("f1", fis1.fieldInfo(0).name);
@@ -269,8 +262,7 @@ public class TestConsistentFieldNumbers 
     writer.forceMerge(1);
     writer.close();
 
-    SegmentInfos sis = new SegmentInfos();
-    sis.read(dir);
+    SegmentInfos sis = SegmentInfos.readLatestCommit(dir);
     for (SegmentCommitInfo si : sis) {
       FieldInfos fis = IndexWriter.readFieldInfos(si);
 

Modified: lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestDeletionPolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestDeletionPolicy.java?rev=1634823&r1=1634822&r2=1634823&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestDeletionPolicy.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestDeletionPolicy.java Tue Oct 28 09:19:21 2014
@@ -285,8 +285,7 @@ public class TestDeletionPolicy extends 
         // if we are on a filesystem that seems to have only
         // 1 second resolution, allow +1 second in commit
         // age tolerance:
-        SegmentInfos sis = new SegmentInfos();
-        sis.read(dir, fileName);
+        SegmentInfos sis = SegmentInfos.readCommit(dir, fileName);
         long modTime = Long.parseLong(sis.getUserData().get("commitTime"));
         oneSecondResolution &= (modTime % 1000) == 0;
         final long leeway = (long) ((SECONDS + (oneSecondResolution ? 1.0:0.0))*1000);

Modified: lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java?rev=1634823&r1=1634822&r2=1634823&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java Tue Oct 28 09:19:21 2014
@@ -644,8 +644,7 @@ public class TestDirectoryReader extends
       addDocumentWithFields(writer);
     writer.close();
 
-    SegmentInfos sis = new SegmentInfos();
-    sis.read(d);
+    SegmentInfos sis = SegmentInfos.readLatestCommit(d);
     DirectoryReader r = DirectoryReader.open(d);
     IndexCommit c = r.getIndexCommit();
 

Modified: lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestDocValuesIndexing.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestDocValuesIndexing.java?rev=1634823&r1=1634822&r2=1634823&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestDocValuesIndexing.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestDocValuesIndexing.java Tue Oct 28 09:19:21 2014
@@ -231,6 +231,7 @@ public class TestDocValuesIndexing exten
     doc.add(new SortedDocValuesField("foo", new BytesRef("hello")));
     try {
       w.addDocument(doc);
+      fail("didn't hit expected exception");
     } catch (IllegalArgumentException iae) {
       // expected
     }
@@ -253,6 +254,7 @@ public class TestDocValuesIndexing exten
     doc.add(new SortedDocValuesField("foo", new BytesRef("hello")));
     try {
       w.addDocument(doc);
+      fail("didn't hit expected exception");
     } catch (IllegalArgumentException iae) {
       // expected
     }
@@ -420,6 +422,7 @@ public class TestDocValuesIndexing exten
     doc.add(new SortedDocValuesField("foo", new BytesRef("hello")));
     try {
       w.addDocument(doc);
+      fail("did not get expected exception");
     } catch (IllegalArgumentException iae) {
       // expected
     }
@@ -456,9 +459,83 @@ public class TestDocValuesIndexing exten
     iwc.setOpenMode(IndexWriterConfig.OpenMode.CREATE);
     w = new IndexWriter(dir, iwc);
     doc = new Document();
+    w.addDocument(doc);
+    w.close();
+    dir.close();
+  }
+
+  public void testMixedTypesAfterReopenAppend1() throws Exception {
+    Directory dir = newDirectory();
+    IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
+    Document doc = new Document();
+    doc.add(new NumericDocValuesField("foo", 0));
+    w.addDocument(doc);
+    w.close();
+
+    w = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
+    doc = new Document();
     doc.add(new SortedDocValuesField("foo", new BytesRef("hello")));
+    try {
+      w.addDocument(doc);
+      fail("did not get expected exception");
+    } catch (IllegalArgumentException iae) {
+      // expected
+    }
+    w.close();
+    dir.close();
+  }
+
+  public void testMixedTypesAfterReopenAppend2() throws IOException {
+    Directory dir = newDirectory();
+    IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random()))) ;
+    Document doc = new Document();
+    doc.add(new SortedSetDocValuesField("foo", new BytesRef("foo")));
     w.addDocument(doc);
     w.close();
+
+    doc = new Document();
+    w = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
+    doc.add(new StringField("foo", "bar", Field.Store.NO));
+    doc.add(new BinaryDocValuesField("foo", new BytesRef("foo")));
+    try {
+      // NOTE: this case follows a different code path inside
+      // DefaultIndexingChain/FieldInfos, because the field (foo)
+      // is first added without DocValues:
+      w.addDocument(doc);
+      fail("did not get expected exception");
+    } catch (IllegalArgumentException iae) {
+      // expected
+    }
+    w.forceMerge(1);
+    w.close();
+    dir.close();
+  }
+
+  public void testMixedTypesAfterReopenAppend3() throws IOException {
+    Directory dir = newDirectory();
+    IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random()))) ;
+    Document doc = new Document();
+    doc.add(new SortedSetDocValuesField("foo", new BytesRef("foo")));
+    w.addDocument(doc);
+    w.close();
+
+    doc = new Document();
+    w = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
+    doc.add(new StringField("foo", "bar", Field.Store.NO));
+    doc.add(new BinaryDocValuesField("foo", new BytesRef("foo")));
+    try {
+      // NOTE: this case follows a different code path inside
+      // DefaultIndexingChain/FieldInfos, because the field (foo)
+      // is first added without DocValues:
+      w.addDocument(doc);
+      fail("did not get expected exception");
+    } catch (IllegalArgumentException iae) {
+      // expected
+    }
+    // Also add another document so there is a segment to write here:
+    w.addDocument(new Document());
+    w.forceMerge(1);
+    w.close();
     dir.close();
   }
 
@@ -528,6 +605,7 @@ public class TestDocValuesIndexing exten
 
     try {
       w.addIndexes(new Directory[] {dir2});
+      fail("didn't hit expected exception");
     } catch (IllegalArgumentException iae) {
       // expected
     }
@@ -535,6 +613,7 @@ public class TestDocValuesIndexing exten
     IndexReader r = DirectoryReader.open(dir2);
     try {
       w.addIndexes(new IndexReader[] {r});
+      fail("didn't hit expected exception");
     } catch (IllegalArgumentException iae) {
       // expected
     }
@@ -819,5 +898,4 @@ public class TestDocValuesIndexing exten
     
     dir.close();
   }
-
 }

Modified: lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestFilterLeafReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestFilterLeafReader.java?rev=1634823&r1=1634822&r2=1634823&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestFilterLeafReader.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestFilterLeafReader.java Tue Oct 28 09:19:21 2014
@@ -166,7 +166,7 @@ public class TestFilterLeafReader extend
     for (Method m : superClazz.getMethods()) {
       final int mods = m.getModifiers();
       if (Modifier.isStatic(mods) || Modifier.isAbstract(mods) || Modifier.isFinal(mods) || m.isSynthetic()
-          || m.getName().equals("attributes")) {
+          || m.getName().equals("attributes") || m.getName().equals("getStats")) {
         continue;
       }
       // The point of these checks is to ensure that methods that have a default

Modified: lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestIndexFileDeleter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestIndexFileDeleter.java?rev=1634823&r1=1634822&r2=1634823&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestIndexFileDeleter.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestIndexFileDeleter.java Tue Oct 28 09:19:21 2014
@@ -85,8 +85,7 @@ public class TestIndexFileDeleter extend
     writer.close();
     
     // read in index to try to not depend on codec-specific filenames so much
-    SegmentInfos sis = new SegmentInfos();
-    sis.read(dir);
+    SegmentInfos sis = SegmentInfos.readLatestCommit(dir);
     SegmentInfo si0 = sis.info(0).info;
     SegmentInfo si1 = sis.info(1).info;
     SegmentInfo si3 = sis.info(3).info;
@@ -123,10 +122,6 @@ public class TestIndexFileDeleter extend
 
     // Create a bogus fnm file when the CFS already exists:
     copyFile(dir, cfsFiles0[0], "_0.fnm");
-    
-    // Create some old segments file:
-    copyFile(dir, "segments_2", "segments");
-    copyFile(dir, "segments_2", "segments_1");
 
     // Create a bogus cfs file shadowing a non-cfs segment:
     
@@ -143,8 +138,7 @@ public class TestIndexFileDeleter extend
     
     String[] filesPre = dir.listAll();
 
-    // Open & close a writer: it should delete the above 4
-    // files and nothing more:
+    // Open & close a writer: it should delete the above files and nothing more:
     writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random()))
                                     .setOpenMode(OpenMode.APPEND));
     writer.close();
@@ -265,8 +259,7 @@ public class TestIndexFileDeleter extend
     // empty commit
     new IndexWriter(dir, new IndexWriterConfig(null)).close();   
     
-    SegmentInfos sis = new SegmentInfos();
-    sis.read(dir);
+    SegmentInfos sis = SegmentInfos.readLatestCommit(dir);
     assertEquals(1, sis.getGeneration());
     
     // no inflation
@@ -283,8 +276,7 @@ public class TestIndexFileDeleter extend
     // empty commit
     new IndexWriter(dir, new IndexWriterConfig(null)).close();   
     
-    SegmentInfos sis = new SegmentInfos();
-    sis.read(dir);
+    SegmentInfos sis = SegmentInfos.readLatestCommit(dir);
     assertEquals(1, sis.getGeneration());
     
     // add trash commit
@@ -308,8 +300,7 @@ public class TestIndexFileDeleter extend
     // empty commit
     new IndexWriter(dir, new IndexWriterConfig(null)).close();   
     
-    SegmentInfos sis = new SegmentInfos();
-    sis.read(dir);
+    SegmentInfos sis = SegmentInfos.readLatestCommit(dir);
     assertEquals(0, sis.counter);
     
     // no inflation
@@ -333,8 +324,7 @@ public class TestIndexFileDeleter extend
     iw.addDocument(new Document());
     iw.commit();
     iw.close();
-    sis = new SegmentInfos();
-    sis.read(dir);
+    sis = SegmentInfos.readLatestCommit(dir);
     assertEquals("_4", sis.info(0).info.name);
     assertEquals(5, sis.counter);
     
@@ -351,8 +341,7 @@ public class TestIndexFileDeleter extend
     iw.close();   
     
     // no deletes: start at 1
-    SegmentInfos sis = new SegmentInfos();
-    sis.read(dir);
+    SegmentInfos sis = SegmentInfos.readLatestCommit(dir);
     assertEquals(1, sis.info(0).getNextDelGen());
     
     // no inflation
@@ -376,8 +365,7 @@ public class TestIndexFileDeleter extend
     // empty commit
     new IndexWriter(dir, new IndexWriterConfig(null)).close();   
     
-    SegmentInfos sis = new SegmentInfos();
-    sis.read(dir);
+    SegmentInfos sis = SegmentInfos.readLatestCommit(dir);
     assertEquals(1, sis.getGeneration());
     
     // add trash file
@@ -400,8 +388,7 @@ public class TestIndexFileDeleter extend
     iw.close();   
     
     // no deletes: start at 1
-    SegmentInfos sis = new SegmentInfos();
-    sis.read(dir);
+    SegmentInfos sis = SegmentInfos.readLatestCommit(dir);
     assertEquals(1, sis.info(0).getNextDelGen());
     
     // add trash file

Modified: lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java?rev=1634823&r1=1634822&r2=1634823&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java Tue Oct 28 09:19:21 2014
@@ -575,8 +575,7 @@ public class TestIndexWriter extends Luc
         writer.addDocument(doc);
       writer.flush(false, true);
       writer.close();
-      SegmentInfos sis = new SegmentInfos();
-      sis.read(dir);
+      SegmentInfos sis = SegmentInfos.readLatestCommit(dir);
       // Since we flushed w/o allowing merging we should now
       // have 10 segments
       assertEquals(10, sis.size());
@@ -2733,8 +2732,7 @@ public class TestIndexWriter extends Luc
     w.addDocument(new Document());
     w.close();
     
-    SegmentInfos sis = new SegmentInfos();
-    sis.read(d);
+    SegmentInfos sis = SegmentInfos.readLatestCommit(d);
     byte[] id1 = sis.getId();
     assertNotNull(id1);
     assertEquals(StringHelper.ID_LENGTH, id1.length);

Modified: lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java?rev=1634823&r1=1634822&r2=1634823&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java Tue Oct 28 09:19:21 2014
@@ -1236,8 +1236,7 @@ public class TestIndexWriterExceptions e
     assertTrue("segment generation should be > 0 but got " + gen, gen > 0);
     
     boolean corrupted = false;
-    SegmentInfos sis = new SegmentInfos();
-    sis.read(dir);
+    SegmentInfos sis = SegmentInfos.readLatestCommit(dir);
     for (SegmentCommitInfo si : sis) {
       assertTrue(si.info.getUseCompoundFile());
       String cfsFiles[] = si.info.getCodec().compoundFormat().files(si.info);
@@ -1314,8 +1313,7 @@ public class TestIndexWriterExceptions e
         w.close();
         IndexReader reader = DirectoryReader.open(dir);
         assertTrue(reader.numDocs() > 0);
-        SegmentInfos sis = new SegmentInfos();
-        sis.read(dir);
+        SegmentInfos sis = SegmentInfos.readLatestCommit(dir);
         for(LeafReaderContext context : reader.leaves()) {
           assertFalse(context.reader().getFieldInfos().hasVectors());
         }
@@ -1682,7 +1680,7 @@ public class TestIndexWriterExceptions e
       if (doFail && name.startsWith("segments_")) {
         StackTraceElement[] trace = new Exception().getStackTrace();
         for (int i = 0; i < trace.length; i++) {
-          if ("read".equals(trace[i].getMethodName())) {
+          if ("readCommit".equals(trace[i].getMethodName()) || "readLatestCommit".equals(trace[i].getMethodName())) {
             throw new UnsupportedOperationException("expected UOE");
           }
         }

Modified: lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterForceMerge.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterForceMerge.java?rev=1634823&r1=1634822&r2=1634823&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterForceMerge.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterForceMerge.java Tue Oct 28 09:19:21 2014
@@ -48,8 +48,7 @@ public class TestIndexWriterForceMerge e
         writer.addDocument(doc);
       writer.close();
 
-      SegmentInfos sis = new SegmentInfos();
-      sis.read(dir);
+      SegmentInfos sis = SegmentInfos.readLatestCommit(dir);
       final int segCount = sis.size();
 
       ldmp = new LogDocMergePolicy();
@@ -59,8 +58,7 @@ public class TestIndexWriterForceMerge e
       writer.forceMerge(3);
       writer.close();
 
-      sis = new SegmentInfos();
-      sis.read(dir);
+      sis = SegmentInfos.readLatestCommit(dir);
       final int optSegCount = sis.size();
 
       if (segCount < 3)
@@ -93,16 +91,14 @@ public class TestIndexWriterForceMerge e
       writer.waitForMerges();
       writer.commit();
 
-      SegmentInfos sis = new SegmentInfos();
-      sis.read(dir);
+      SegmentInfos sis = SegmentInfos.readLatestCommit(dir);
 
       final int segCount = sis.size();
       writer.forceMerge(7);
       writer.commit();
       writer.waitForMerges();
 
-      sis = new SegmentInfos();
-      sis.read(dir);
+      sis = SegmentInfos.readLatestCommit(dir);
       final int optSegCount = sis.size();
 
       if (segCount < 7)
@@ -226,8 +222,7 @@ public class TestIndexWriterForceMerge e
         assertTrue(reader.leaves().size() > 1);
         reader.close();
 
-        SegmentInfos infos = new SegmentInfos();
-        infos.read(dir);
+        SegmentInfos infos = SegmentInfos.readLatestCommit(dir);
         assertEquals(2, infos.size());
       }
     }

Modified: lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterThreadsToSegments.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterThreadsToSegments.java?rev=1634823&r1=1634822&r2=1634823&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterThreadsToSegments.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterThreadsToSegments.java Tue Oct 28 09:19:21 2014
@@ -34,8 +34,10 @@ import org.apache.lucene.document.TextFi
 import org.apache.lucene.store.AlreadyClosedException;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexInput;
 import org.apache.lucene.util.IOUtils;
 import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.StringHelper;
 import org.apache.lucene.util.TestUtil;
 
 public class TestIndexWriterThreadsToSegments extends LuceneTestCase {
@@ -331,7 +333,8 @@ public class TestIndexWriterThreadsToSeg
           String segName = IndexFileNames.parseSegmentName(fileName);
           if (segSeen.contains(segName) == false) {
             segSeen.add(segName);
-            SegmentInfo si = TestUtil.getDefaultCodec().segmentInfoFormat().read(dir, segName, IOContext.DEFAULT);
+            byte id[] = readSegmentInfoID(dir, fileName);
+            SegmentInfo si = TestUtil.getDefaultCodec().segmentInfoFormat().read(dir, segName, id, IOContext.DEFAULT);
             si.setCodec(codec);
             SegmentCommitInfo sci = new SegmentCommitInfo(si, 0, -1, -1, -1);
             SegmentReader sr = new SegmentReader(sci, IOContext.DEFAULT);
@@ -349,4 +352,17 @@ public class TestIndexWriterThreadsToSeg
     w.close();
     dir.close();
   }
+  
+  // TODO: remove this hack and fix this test to be better?
+  // the whole thing relies on default codec too...
+  byte[] readSegmentInfoID(Directory dir, String file) throws IOException {
+    try (IndexInput in = dir.openInput(file, IOContext.DEFAULT)) {
+      in.readInt(); // magic
+      in.readString(); // codec name
+      in.readInt(); // version
+      byte id[] = new byte[StringHelper.ID_LENGTH];
+      in.readBytes(id, 0, id.length);
+      return id;
+    }
+  }
 }

Modified: lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestRollingUpdates.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestRollingUpdates.java?rev=1634823&r1=1634822&r2=1634823&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestRollingUpdates.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestRollingUpdates.java Tue Oct 28 09:19:21 2014
@@ -143,8 +143,7 @@ public class TestRollingUpdates extends 
     docs.close();
     
     // LUCENE-4455:
-    SegmentInfos infos = new SegmentInfos();
-    infos.read(dir);
+    SegmentInfos infos = SegmentInfos.readLatestCommit(dir);
     long totalBytes = 0;
     for(SegmentCommitInfo sipc : infos) {
       totalBytes += sipc.sizeInBytes();



Mime
View raw message