lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sar...@apache.org
Subject svn commit: r1140809 [5/11] - in /lucene/dev/branches/solr2452: ./ dev-tools/eclipse/ dev-tools/idea/.idea/ dev-tools/idea/lucene/contrib/ dev-tools/idea/lucene/contrib/demo/ dev-tools/idea/lucene/contrib/highlighter/ dev-tools/idea/lucene/contrib/inst...
Date Tue, 28 Jun 2011 19:42:44 GMT
Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/SortField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/SortField.java?rev=1140809&r1=1140808&r2=1140809&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/SortField.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/SortField.java Tue Jun 28 19:42:31 2011
@@ -21,8 +21,8 @@ import java.io.IOException;
 import java.util.Comparator;
 
 import org.apache.lucene.search.cache.*;
-import org.apache.lucene.util.StringHelper;
 import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.StringHelper;
 
 // TODO(simonw) -- for cleaner transition, maybe we should make
 // a new SortField that subclasses this one and always uses
@@ -39,69 +39,70 @@ import org.apache.lucene.util.BytesRef;
  */
 public class SortField {
 
-  /** Sort by document score (relevance).  Sort values are Float and higher
-   * values are at the front. */
-  public static final int SCORE = 0;
-
-  /** Sort by document number (index order).  Sort values are Integer and lower
-   * values are at the front. */
-  public static final int DOC = 1;
-
-  // reserved, in Lucene 2.9, there was a constant: AUTO = 2;
-
-  /** Sort using term values as Strings.  Sort values are String and lower
-   * values are at the front. */
-  public static final int STRING = 3;
-
-  /** Sort using term values as encoded Integers.  Sort values are Integer and
-   * lower values are at the front. */
-  public static final int INT = 4;
-
-  /** Sort using term values as encoded Floats.  Sort values are Float and
-   * lower values are at the front. */
-  public static final int FLOAT = 5;
-
-  /** Sort using term values as encoded Longs.  Sort values are Long and
-   * lower values are at the front. */
-  public static final int LONG = 6;
-
-  /** Sort using term values as encoded Doubles.  Sort values are Double and
-   * lower values are at the front. */
-  public static final int DOUBLE = 7;
-
-  /** Sort using term values as encoded Shorts.  Sort values are Short and
-   * lower values are at the front. */
-  public static final int SHORT = 8;
-
-  /** Sort using a custom Comparator.  Sort values are any Comparable and
-   * sorting is done according to natural order. */
-  public static final int CUSTOM = 9;
-
-  /** Sort using term values as encoded Bytes.  Sort values are Byte and
-   * lower values are at the front. */
-  public static final int BYTE = 10;
-  
-  /** Sort using term values as Strings, but comparing by
-   * value (using String.compareTo) for all comparisons.
-   * This is typically slower than {@link #STRING}, which
-   * uses ordinals to do the sorting. */
-  public static final int STRING_VAL = 11;
-
-  /** Sort use byte[] index values. */
-  public static final int BYTES = 12;
-
-  /** Force rewriting of SortField using {@link SortField#rewrite(IndexSearcher)}
-   * before it can be used for sorting */
-  public static final int REWRITEABLE = 13;
+  public static enum Type {
+
+    /** Sort by document score (relevance).  Sort values are Float and higher
+     * values are at the front. */
+    SCORE,
+
+    /** Sort by document number (index order).  Sort values are Integer and lower
+     * values are at the front. */
+    DOC,
+
+    /** Sort using term values as Strings.  Sort values are String and lower
+     * values are at the front. */
+    STRING,
+
+    /** Sort using term values as encoded Integers.  Sort values are Integer and
+     * lower values are at the front. */
+    INT,
+
+    /** Sort using term values as encoded Floats.  Sort values are Float and
+     * lower values are at the front. */
+    FLOAT,
+
+    /** Sort using term values as encoded Longs.  Sort values are Long and
+     * lower values are at the front. */
+    LONG,
+
+    /** Sort using term values as encoded Doubles.  Sort values are Double and
+     * lower values are at the front. */
+    DOUBLE,
+
+    /** Sort using term values as encoded Shorts.  Sort values are Short and
+     * lower values are at the front. */
+    SHORT,
+
+    /** Sort using a custom Comparator.  Sort values are any Comparable and
+     * sorting is done according to natural order. */
+    CUSTOM,
+
+    /** Sort using term values as encoded Bytes.  Sort values are Byte and
+     * lower values are at the front. */
+    BYTE,
+
+    /** Sort using term values as Strings, but comparing by
+     * value (using String.compareTo) for all comparisons.
+     * This is typically slower than {@link #STRING}, which
+     * uses ordinals to do the sorting. */
+    STRING_VAL,
+
+    /** Sort use byte[] index values. */
+    BYTES,
+
+    /** Force rewriting of SortField using {@link SortField#rewrite(IndexSearcher)}
+     * before it can be used for sorting */
+    REWRITEABLE
+  }
 
   /** Represents sorting by document score (relevance). */
-  public static final SortField FIELD_SCORE = new SortField(null, SCORE);
+  public static final SortField FIELD_SCORE = new SortField(null, Type.SCORE);
 
   /** Represents sorting by document number (index order). */
-  public static final SortField FIELD_DOC = new SortField(null, DOC);
+  public static final SortField FIELD_DOC = new SortField(null, Type.DOC);
 
   private String field;
-  private int type;  // defaults to determining type dynamically
+  private Type type;  // defaults to determining type dynamically
   boolean reverse = false;  // defaults to natural order
   private CachedArrayCreator<?> creator;
   public Object missingValue = null; // used for 'sortMissingFirst/Last'
@@ -115,7 +116,7 @@ public class SortField {
    *               <code>type</code> is SCORE or DOC.
    * @param type   Type of values in the terms.
    */
-  public SortField(String field, int type) {
+  public SortField(String field, Type type) {
     initFieldType(field, type);
   }
 
@@ -126,7 +127,7 @@ public class SortField {
    * @param type   Type of values in the terms.
    * @param reverse True if natural order should be reversed.
    */
-  public SortField(String field, int type, boolean reverse) {
+  public SortField(String field, Type type, boolean reverse) {
     initFieldType(field, type);
     this.reverse = reverse;
   }
@@ -166,36 +167,31 @@ public class SortField {
     if (field == null) {
       throw new IllegalArgumentException("field can only be null when type is SCORE or DOC");
     } 
-    this.field = StringHelper.intern(field);
+    this.field = field;
     this.reverse = reverse;
     
     if (parser instanceof FieldCache.IntParser) {
-      this.type = INT;
       this.creator = new IntValuesCreator( field, (FieldCache.IntParser)parser );
     }
     else if (parser instanceof FieldCache.FloatParser) {
-      this.type = FLOAT;
       this.creator = new FloatValuesCreator( field, (FieldCache.FloatParser)parser );
     }
     else if (parser instanceof FieldCache.ShortParser) {
-      this.type = SHORT;
       this.creator = new ShortValuesCreator( field, (FieldCache.ShortParser)parser );
     }
     else if (parser instanceof FieldCache.ByteParser) {
-      this.type = BYTE;
       this.creator = new ByteValuesCreator( field, (FieldCache.ByteParser)parser );
     }
     else if (parser instanceof FieldCache.LongParser) {
-      this.type = LONG;
       this.creator = new LongValuesCreator( field, (FieldCache.LongParser)parser );
     }
     else if (parser instanceof FieldCache.DoubleParser) {
-      this.type = DOUBLE;
       this.creator = new DoubleValuesCreator( field, (FieldCache.DoubleParser)parser );
     }
     else
       throw new IllegalArgumentException("Parser instance does not subclass existing numeric parser from FieldCache (got " + parser + ")");
 
+    this.type = this.creator.getSortType();
   }
   
   /**
@@ -205,10 +201,10 @@ public class SortField {
    */
   public SortField( CachedArrayCreator<?> creator, boolean reverse ) 
   {
-    this.field = StringHelper.intern(creator.field);
+    this.field = creator.field;
     this.reverse = reverse;
     this.creator = creator;
-    this.type = creator.getSortTypeID();
+    this.type = creator.getSortType();
   }
   
   public SortField setMissingValue( Object v )
@@ -230,7 +226,7 @@ public class SortField {
    * @param comparator Returns a comparator for sorting hits.
    */
   public SortField(String field, FieldComparatorSource comparator) {
-    initFieldType(field, CUSTOM);
+    initFieldType(field, Type.CUSTOM);
     this.comparatorSource = comparator;
   }
 
@@ -240,20 +236,20 @@ public class SortField {
    * @param reverse True if natural order should be reversed.
    */
   public SortField(String field, FieldComparatorSource comparator, boolean reverse) {
-    initFieldType(field, CUSTOM);
+    initFieldType(field, Type.CUSTOM);
     this.reverse = reverse;
     this.comparatorSource = comparator;
   }
 
   // Sets field & type, and ensures field is not NULL unless
   // type is SCORE or DOC
-  private void initFieldType(String field, int type) {
+  private void initFieldType(String field, Type type) {
     this.type = type;
     if (field == null) {
-      if (type != SCORE && type != DOC)
+      if (type != Type.SCORE && type != Type.DOC)
         throw new IllegalArgumentException("field can only be null when type is SCORE or DOC");
     } else {
-      this.field = StringHelper.intern(field);
+      this.field = field;
     }
     
     if( creator != null ) {
@@ -280,7 +276,7 @@ public class SortField {
   /** Returns the type of contents in the field.
    * @return One of the constants SCORE, DOC, STRING, INT or FLOAT.
    */
-  public int getType() {
+  public Type getType() {
     return type;
   }
 
@@ -359,6 +355,10 @@ public class SortField {
       case CUSTOM:
         buffer.append("<custom:\"").append(field).append("\": ").append(comparatorSource).append('>');
         break;
+      
+      case REWRITEABLE:
+        buffer.append("<rewriteable: \"").append(field).append("\">");
+        break;
 
       default:
         buffer.append("<???: \"").append(field).append("\">");
@@ -381,7 +381,7 @@ public class SortField {
     if (!(o instanceof SortField)) return false;
     final SortField other = (SortField)o;
     return (
-      other.field == this.field // field is always interned
+      StringHelper.equals(other.field, this.field)
       && other.type == this.type
       && other.reverse == this.reverse
       && (other.comparatorSource == null ? this.comparatorSource == null : other.comparatorSource.equals(this.comparatorSource))
@@ -396,7 +396,7 @@ public class SortField {
    *  used). */
   @Override
   public int hashCode() {
-    int hash=type^0x346565dd + Boolean.valueOf(reverse).hashCode()^0xaf5998bb;
+    int hash = type.hashCode() ^ 0x346565dd + Boolean.valueOf(reverse).hashCode() ^ 0xaf5998bb;
     if (field != null) hash += field.hashCode()^0xff5685dd;
     if (comparatorSource != null) hash += comparatorSource.hashCode();
     if (creator != null) hash += creator.hashCode()^0x3aaf56ff;
@@ -438,49 +438,49 @@ public class SortField {
   public FieldComparator getComparator(final int numHits, final int sortPos) throws IOException {
 
     switch (type) {
-    case SortField.SCORE:
+    case SCORE:
       return new FieldComparator.RelevanceComparator(numHits);
 
-    case SortField.DOC:
+    case DOC:
       return new FieldComparator.DocComparator(numHits);
 
-    case SortField.INT:
+    case INT:
       if (useIndexValues) {
         return new FieldComparator.IntDocValuesComparator(numHits, field);
       } else {
         return new FieldComparator.IntComparator(numHits, (IntValuesCreator)creator, (Integer) missingValue);
       }
 
-    case SortField.FLOAT:
+    case FLOAT:
       if (useIndexValues) {
         return new FieldComparator.FloatDocValuesComparator(numHits, field);
       } else {
         return new FieldComparator.FloatComparator(numHits, (FloatValuesCreator) creator, (Float) missingValue);
       }
 
-    case SortField.LONG:
+    case LONG:
       return new FieldComparator.LongComparator(numHits, (LongValuesCreator)creator, (Long)missingValue );
 
-    case SortField.DOUBLE:
+    case DOUBLE:
       return new FieldComparator.DoubleComparator(numHits, (DoubleValuesCreator)creator, (Double)missingValue );
 
-    case SortField.BYTE:
+    case BYTE:
       return new FieldComparator.ByteComparator(numHits, (ByteValuesCreator)creator, (Byte)missingValue );
 
-    case SortField.SHORT:
+    case SHORT:
       return new FieldComparator.ShortComparator(numHits, (ShortValuesCreator)creator, (Short)missingValue );
 
-    case SortField.CUSTOM:
+    case CUSTOM:
       assert comparatorSource != null;
       return comparatorSource.newComparator(field, numHits, sortPos, reverse);
 
-    case SortField.STRING:
+    case STRING:
       return new FieldComparator.TermOrdValComparator(numHits, field, sortPos, reverse);
 
-    case SortField.STRING_VAL:
+    case STRING_VAL:
       return new FieldComparator.TermValComparator(numHits, field);
 
-    case SortField.REWRITEABLE:
+    case REWRITEABLE:
       throw new IllegalStateException("SortField needs to be rewritten through Sort.rewrite(..) and SortField.rewrite(..)");
         
     default:
@@ -491,7 +491,7 @@ public class SortField {
   /**
    * Rewrites this SortField, returning a new SortField if a change is made.
    * Subclasses should override this define their rewriting behavior when this
-   * SortField is of type {@link SortField#REWRITEABLE}
+   * SortField is of type {@link SortField.Type#REWRITEABLE}
    *
    * @param searcher IndexSearcher to use during rewriting
    * @return New rewritten SortField, or {@code this} if nothing has changed.

Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/TopTermsRewrite.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/TopTermsRewrite.java?rev=1140809&r1=1140808&r2=1140809&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/TopTermsRewrite.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/TopTermsRewrite.java Tue Jun 28 19:42:31 2011
@@ -131,12 +131,11 @@ public abstract class TopTermsRewrite<Q 
       }
     });
     
-    final Term placeholderTerm = new Term(query.field);
     final Q q = getTopLevelQuery();
     final ScoreTerm[] scoreTerms = stQueue.toArray(new ScoreTerm[stQueue.size()]);
     ArrayUtil.mergeSort(scoreTerms, scoreTermSortByTermComp);
     for (final ScoreTerm st : scoreTerms) {
-      final Term term = placeholderTerm.createTerm(st.bytes);
+      final Term term = new Term(query.field, st.bytes);
       assert reader.docFreq(term) == st.termState.docFreq() : "reader DF is " + reader.docFreq(term) + " vs " + st.termState.docFreq();
       addClause(q, term, st.termState.docFreq(), query.getBoost() * st.boost, st.termState); // add to query
     }

Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/cache/ByteValuesCreator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/cache/ByteValuesCreator.java?rev=1140809&r1=1140808&r2=1140809&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/cache/ByteValuesCreator.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/cache/ByteValuesCreator.java Tue Jun 28 19:42:31 2011
@@ -61,8 +61,8 @@ public class ByteValuesCreator extends C
   }
   
   @Override
-  public int getSortTypeID() {
-    return SortField.BYTE;
+  public SortField.Type getSortType() {
+    return SortField.Type.BYTE;
   }
 
   //--------------------------------------------------------------------------------
@@ -143,4 +143,4 @@ public class ByteValuesCreator extends C
       vals.valid = new Bits.MatchNoBits( maxDoc );
     }
   }
-}
\ No newline at end of file
+}

Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/cache/CachedArrayCreator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/cache/CachedArrayCreator.java?rev=1140809&r1=1140808&r2=1140809&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/cache/CachedArrayCreator.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/cache/CachedArrayCreator.java Tue Jun 28 19:42:31 2011
@@ -26,6 +26,7 @@ import org.apache.lucene.index.Terms;
 import org.apache.lucene.index.TermsEnum;
 import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.search.FieldCache.Parser;
+import org.apache.lucene.search.SortField;
 import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.OpenBitSet;
@@ -73,7 +74,7 @@ public abstract class CachedArrayCreator
   /** Return the type that the array will hold */
   public abstract Class getArrayType();
   public abstract Parser getParser();
-  public abstract int getSortTypeID();
+  public abstract SortField.Type getSortType();
 
   protected void setParserAndResetCounts(T value, Parser parser)
   {

Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/cache/DocTermsCreator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/cache/DocTermsCreator.java?rev=1140809&r1=1140808&r2=1140809&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/cache/DocTermsCreator.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/cache/DocTermsCreator.java Tue Jun 28 19:42:31 2011
@@ -29,7 +29,6 @@ import org.apache.lucene.search.FieldCac
 import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.PagedBytes;
-import org.apache.lucene.util.StringHelper;
 import org.apache.lucene.util.packed.GrowableWriter;
 import org.apache.lucene.util.packed.PackedInts;
 
@@ -66,7 +65,6 @@ public class DocTermsCreator extends Ent
   @Override
   public DocTerms create(IndexReader reader) throws IOException {
 
-    String field = StringHelper.intern(this.field); // TODO?? necessary?
     Terms terms = MultiFields.getTerms(reader, field);
 
     final boolean fasterButMoreRAM = hasOption( FASTER_BUT_MORE_RAM );

Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/cache/DocTermsIndexCreator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/cache/DocTermsIndexCreator.java?rev=1140809&r1=1140808&r2=1140809&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/cache/DocTermsIndexCreator.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/cache/DocTermsIndexCreator.java Tue Jun 28 19:42:31 2011
@@ -34,7 +34,6 @@ import org.apache.lucene.util.ArrayUtil;
 import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.PagedBytes;
-import org.apache.lucene.util.StringHelper;
 import org.apache.lucene.util.packed.GrowableWriter;
 import org.apache.lucene.util.packed.PackedInts;
 
@@ -70,7 +69,6 @@ public class DocTermsIndexCreator extend
   @Override
   public DocTermsIndex create(IndexReader reader) throws IOException
   {
-    String field = StringHelper.intern(this.field); // TODO?? necessary?
     Terms terms = MultiFields.getTerms(reader, field);
 
     final boolean fasterButMoreRAM = hasOption(FASTER_BUT_MORE_RAM);
@@ -241,13 +239,13 @@ public class DocTermsIndexCreator extend
       }
 
       @Override
-      public SeekStatus seek(BytesRef text, boolean useCache) throws IOException {
+      public SeekStatus seekCeil(BytesRef text, boolean useCache /* ignored */) throws IOException {
         int low = 1;
         int high = numOrd-1;
         
         while (low <= high) {
           int mid = (low + high) >>> 1;
-          seek(mid);
+          seekExact(mid);
           int cmp = term.compareTo(text);
 
           if (cmp < 0)
@@ -261,19 +259,17 @@ public class DocTermsIndexCreator extend
         if (low == numOrd) {
           return SeekStatus.END;
         } else {
-          seek(low);
+          seekExact(low);
           return SeekStatus.NOT_FOUND;
         }
       }
 
-      @Override
-      public SeekStatus seek(long ord) throws IOException {
+      public void seekExact(long ord) throws IOException {
         assert(ord >= 0 && ord <= numOrd);
         // TODO: if gap is small, could iterate from current position?  Or let user decide that?
         currentBlockNumber = bytes.fillAndGetIndex(term, termOrdToBytesOffset.get((int)ord));
         end = blockEnds[currentBlockNumber];
         currentOrd = (int)ord;
-        return SeekStatus.FOUND;
       }
 
       @Override
@@ -341,9 +337,9 @@ public class DocTermsIndexCreator extend
       }
 
       @Override
-      public void seek(BytesRef term, TermState state) throws IOException {
+      public void seekExact(BytesRef term, TermState state) throws IOException {
         assert state != null && state instanceof OrdTermState;
-        this.seek(((OrdTermState)state).ord);
+        this.seekExact(((OrdTermState)state).ord);
       }
 
       @Override

Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/cache/DoubleValuesCreator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/cache/DoubleValuesCreator.java?rev=1140809&r1=1140808&r2=1140809&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/cache/DoubleValuesCreator.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/cache/DoubleValuesCreator.java Tue Jun 28 19:42:31 2011
@@ -61,8 +61,8 @@ public class DoubleValuesCreator extends
   }
   
   @Override
-  public int getSortTypeID() {
-    return SortField.DOUBLE;
+  public SortField.Type getSortType() {
+    return SortField.Type.DOUBLE;
   }
 
   //--------------------------------------------------------------------------------
@@ -161,4 +161,4 @@ public class DoubleValuesCreator extends
       vals.valid = new Bits.MatchNoBits( maxDoc );
     }
   }
-}
\ No newline at end of file
+}

Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/cache/FloatValuesCreator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/cache/FloatValuesCreator.java?rev=1140809&r1=1140808&r2=1140809&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/cache/FloatValuesCreator.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/cache/FloatValuesCreator.java Tue Jun 28 19:42:31 2011
@@ -61,8 +61,8 @@ public class FloatValuesCreator extends 
   }
   
   @Override
-  public int getSortTypeID() {
-    return SortField.FLOAT;
+  public SortField.Type getSortType() {
+    return SortField.Type.FLOAT;
   }
 
 
@@ -162,4 +162,4 @@ public class FloatValuesCreator extends 
       vals.valid = new Bits.MatchNoBits( maxDoc );
     }
   }
-}
\ No newline at end of file
+}

Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/cache/IntValuesCreator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/cache/IntValuesCreator.java?rev=1140809&r1=1140808&r2=1140809&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/cache/IntValuesCreator.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/cache/IntValuesCreator.java Tue Jun 28 19:42:31 2011
@@ -61,8 +61,8 @@ public class IntValuesCreator extends Ca
   }
   
   @Override
-  public int getSortTypeID() {
-    return SortField.INT;
+  public SortField.Type getSortType() {
+    return SortField.Type.INT;
   }
 
 
@@ -162,4 +162,4 @@ public class IntValuesCreator extends Ca
       vals.valid = new Bits.MatchNoBits( maxDoc );
     }
   }
-}
\ No newline at end of file
+}

Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/cache/LongValuesCreator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/cache/LongValuesCreator.java?rev=1140809&r1=1140808&r2=1140809&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/cache/LongValuesCreator.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/cache/LongValuesCreator.java Tue Jun 28 19:42:31 2011
@@ -61,8 +61,8 @@ public class LongValuesCreator extends C
   }
   
   @Override
-  public int getSortTypeID() {
-    return SortField.LONG;
+  public SortField.Type getSortType() {
+    return SortField.Type.LONG;
   }
 
 
@@ -162,4 +162,4 @@ public class LongValuesCreator extends C
       vals.valid = new Bits.MatchNoBits( maxDoc );
     }
   }
-}
\ No newline at end of file
+}

Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/cache/ShortValuesCreator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/cache/ShortValuesCreator.java?rev=1140809&r1=1140808&r2=1140809&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/cache/ShortValuesCreator.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/cache/ShortValuesCreator.java Tue Jun 28 19:42:31 2011
@@ -61,8 +61,8 @@ public class ShortValuesCreator extends 
   }
   
   @Override
-  public int getSortTypeID() {
-    return SortField.SHORT;
+  public SortField.Type getSortType() {
+    return SortField.Type.SHORT;
   }
 
 
@@ -144,4 +144,4 @@ public class ShortValuesCreator extends 
       vals.valid = new Bits.MatchNoBits( maxDoc );
     }
   }
-}
\ No newline at end of file
+}

Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/function/NumericIndexDocValueSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/function/NumericIndexDocValueSource.java?rev=1140809&r1=1140808&r2=1140809&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/function/NumericIndexDocValueSource.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/function/NumericIndexDocValueSource.java Tue Jun 28 19:42:31 2011
@@ -59,7 +59,7 @@ public class NumericIndexDocValueSource 
         }
       };
 
-    case INTS:
+    case VAR_INTS:
       return new DocValues() {
         @Override
         public String toString(int doc) {

Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/store/ByteArrayDataInput.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/store/ByteArrayDataInput.java?rev=1140809&r1=1140808&r2=1140809&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/store/ByteArrayDataInput.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/store/ByteArrayDataInput.java Tue Jun 28 19:42:31 2011
@@ -17,6 +17,8 @@ package org.apache.lucene.store;
  * limitations under the License.
  */
 
+import org.apache.lucene.util.BytesRef;
+
 /** @lucene.experimental */
 public final class ByteArrayDataInput extends DataInput {
 
@@ -25,9 +27,16 @@ public final class ByteArrayDataInput ex
   private int pos;
   private int limit;
 
-  // TODO: allow BytesRef (slice) too
   public ByteArrayDataInput(byte[] bytes) {
-    this.bytes = bytes;
+    reset(bytes);
+  }
+
+  public ByteArrayDataInput(byte[] bytes, int offset, int len) {
+    reset(bytes, offset, len);
+  }
+
+  public ByteArrayDataInput() {
+    reset(BytesRef.EMPTY_BYTES);
   }
 
   public void reset(byte[] bytes) {
@@ -41,7 +50,7 @@ public final class ByteArrayDataInput ex
   public void reset(byte[] bytes, int offset, int len) {
     this.bytes = bytes;
     pos = offset;
-    limit = len;
+    limit = offset + len;
   }
 
   public boolean eof() {
@@ -59,12 +68,14 @@ public final class ByteArrayDataInput ex
  
   @Override
   public int readInt() {
+    assert pos+4 <= limit;
     return ((bytes[pos++] & 0xFF) << 24) | ((bytes[pos++] & 0xFF) << 16)
       | ((bytes[pos++] & 0xFF) <<  8) |  (bytes[pos++] & 0xFF);
   }
  
   @Override
   public long readLong() {
+    assert pos+8 <= limit;
     final int i1 = ((bytes[pos++] & 0xff) << 24) | ((bytes[pos++] & 0xff) << 16) |
       ((bytes[pos++] & 0xff) << 8) | (bytes[pos++] & 0xff);
     final int i2 = ((bytes[pos++] & 0xff) << 24) | ((bytes[pos++] & 0xff) << 16) |
@@ -74,9 +85,11 @@ public final class ByteArrayDataInput ex
 
   @Override
   public int readVInt() {
+    checkBounds();
     byte b = bytes[pos++];
     int i = b & 0x7F;
     for (int shift = 7; (b & 0x80) != 0; shift += 7) {
+      checkBounds();
       b = bytes[pos++];
       i |= (b & 0x7F) << shift;
     }
@@ -85,9 +98,11 @@ public final class ByteArrayDataInput ex
  
   @Override
   public long readVLong() {
+    checkBounds();
     byte b = bytes[pos++];
     long i = b & 0x7F;
     for (int shift = 7; (b & 0x80) != 0; shift += 7) {
+      checkBounds();
       b = bytes[pos++];
       i |= (b & 0x7FL) << shift;
     }
@@ -97,7 +112,7 @@ public final class ByteArrayDataInput ex
   // NOTE: AIOOBE not EOF if you read too much
   @Override
   public byte readByte() {
-    assert pos < limit;
+    checkBounds();
     return bytes[pos++];
   }
 
@@ -108,4 +123,9 @@ public final class ByteArrayDataInput ex
     System.arraycopy(bytes, pos, b, offset, len);
     pos += len;
   }
+
+  private boolean checkBounds() {
+    assert pos < limit;
+    return true;
+  }
 }

Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/store/DataOutput.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/store/DataOutput.java?rev=1140809&r1=1140808&r2=1140809&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/store/DataOutput.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/store/DataOutput.java Tue Jun 28 19:42:31 2011
@@ -60,6 +60,14 @@ public abstract class DataOutput {
     writeByte((byte)(i >>  8));
     writeByte((byte) i);
   }
+  
+  /** Writes a short as two bytes.
+   * @see DataInput#readShort()
+   */
+  public void writeShort(short i) throws IOException {
+    writeByte((byte)(i >>  8));
+    writeByte((byte) i);
+  }
 
   /** Writes an int in a variable-length format.  Writes between one and
    * five bytes.  Smaller values take fewer bytes.  Negative numbers are not

Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/store/Directory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/store/Directory.java?rev=1140809&r1=1140808&r2=1140809&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/store/Directory.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/store/Directory.java Tue Jun 28 19:42:31 2011
@@ -112,11 +112,35 @@ public abstract class Directory implemen
    * implementation may ignore the buffer size.  Currently
    * the only Directory implementations that respect this
    * parameter are {@link FSDirectory} and {@link
-   * org.apache.lucene.index.CompoundFileReader}.
+   * CompoundFileDirectory}.
   */
   public IndexInput openInput(String name, int bufferSize) throws IOException {
     return openInput(name);
   }
+  
+  /** 
+   * Returns a {@link CompoundFileDirectory} capable of
+   * reading the Lucene compound file format.  
+   * <p>
+   * The default implementation returns 
+   * {@link DefaultCompoundFileDirectory}.
+   * @lucene.experimental
+   */
+  public CompoundFileDirectory openCompoundInput(String name, int bufferSize) throws IOException {
+    return new DefaultCompoundFileDirectory(this, name, bufferSize, false);
+  }
+  
+  /** 
+   * Returns a {@link CompoundFileDirectory} capable of
+   * writing the Lucene compound file format.  
+   * <p>
+   * The default implementation returns 
+   * {@link DefaultCompoundFileDirectory}.
+   * @lucene.experimental
+   */
+  public CompoundFileDirectory createCompoundOutput(String name) throws IOException {
+    return new DefaultCompoundFileDirectory(this, name, 1024, true);
+  }
 
   /** Construct a {@link Lock}.
    * @param name the name of the lock file
@@ -200,10 +224,12 @@ public abstract class Directory implemen
    * overwrite it if it does.
    */
   public void copy(Directory to, String src, String dest) throws IOException {
-    IndexOutput os = to.createOutput(dest);
-    IndexInput is = openInput(src);
+    IndexOutput os = null;
+    IndexInput is = null;
     IOException priorException = null;
     try {
+      os = to.createOutput(dest);
+      is = openInput(src);
       is.copyBytes(os, is.length());
     } catch (IOException ioe) {
       priorException = ioe;

Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/store/FileSwitchDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/store/FileSwitchDirectory.java?rev=1140809&r1=1140808&r2=1140809&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/store/FileSwitchDirectory.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/store/FileSwitchDirectory.java Tue Jun 28 19:42:31 2011
@@ -148,4 +148,14 @@ public class FileSwitchDirectory extends
   public IndexInput openInput(String name) throws IOException {
     return getDirectory(name).openInput(name);
   }
+
+  @Override
+  public CompoundFileDirectory openCompoundInput(String name, int bufferSize) throws IOException {
+    return getDirectory(name).openCompoundInput(name, bufferSize);
+  }
+  
+  @Override
+  public CompoundFileDirectory createCompoundOutput(String name) throws IOException {
+    return getDirectory(name).createCompoundOutput(name);
+  }
 }

Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/store/IndexOutput.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/store/IndexOutput.java?rev=1140809&r1=1140808&r2=1140809&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/store/IndexOutput.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/store/IndexOutput.java Tue Jun 28 19:42:31 2011
@@ -58,4 +58,5 @@ public abstract class IndexOutput extend
    * @param length file length
    */
   public void setLength(long length) throws IOException {}
+
 }

Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/store/MMapDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/store/MMapDirectory.java?rev=1140809&r1=1140808&r2=1140809&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/store/MMapDirectory.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/store/MMapDirectory.java Tue Jun 28 19:42:31 2011
@@ -32,6 +32,7 @@ import java.security.PrivilegedActionExc
 import java.lang.reflect.Method;
 
 import org.apache.lucene.util.Constants;
+import org.apache.lucene.util.IOUtils;
 
 /** File-based {@link Directory} implementation that uses
  *  mmap for reading, and {@link
@@ -213,11 +214,49 @@ public class MMapDirectory extends FSDir
     File f = new File(getDirectory(), name);
     RandomAccessFile raf = new RandomAccessFile(f, "r");
     try {
-      return new MMapIndexInput(raf, chunkSizePower);
+      return new MMapIndexInput(raf, 0, raf.length(), chunkSizePower);
     } finally {
       raf.close();
     }
   }
+  
+  @Override
+  public CompoundFileDirectory openCompoundInput(String name, int bufferSize) throws IOException {
+    return new MMapCompoundFileDirectory(name, bufferSize);
+  }
+  
+  private final class MMapCompoundFileDirectory extends CompoundFileDirectory {
+    private RandomAccessFile raf = null;
+
+    public MMapCompoundFileDirectory(String fileName, int readBufferSize) throws IOException {
+      super(MMapDirectory.this, fileName, readBufferSize);
+      IndexInput stream = null;
+      try {
+        File f = new File(MMapDirectory.this.getDirectory(), fileName);
+        raf = new RandomAccessFile(f, "r");
+        stream = new MMapIndexInput(raf, 0, raf.length(), chunkSizePower);
+        initForRead(CompoundFileDirectory.readEntries(stream, MMapDirectory.this, fileName));
+        stream.close();
+      } catch (IOException e) {
+        // throw our original exception
+        IOUtils.closeSafely(e, raf, stream);
+      }
+    }
+
+    @Override
+    public IndexInput openInputSlice(String id, long offset, long length, int readBufferSize) throws IOException {
+      return new MMapIndexInput(raf, offset, length, chunkSizePower);
+    }
+
+    @Override
+    public synchronized void close() throws IOException {
+      try {
+        raf.close();
+      } finally {
+        super.close();
+      }
+    }
+  }
 
   // Because Java's ByteBuffer uses an int to address the
   // values, it's necessary to access a file >
@@ -235,8 +274,8 @@ public class MMapDirectory extends FSDir
   
     private boolean isClone = false;
     
-    MMapIndexInput(RandomAccessFile raf, int chunkSizePower) throws IOException {
-      this.length = raf.length();
+    MMapIndexInput(RandomAccessFile raf, long offset, long length, int chunkSizePower) throws IOException {
+      this.length = length;
       this.chunkSizePower = chunkSizePower;
       this.chunkSize = 1L << chunkSizePower;
       this.chunkSizeMask = chunkSize - 1L;
@@ -261,7 +300,7 @@ public class MMapDirectory extends FSDir
           ? chunkSize
           : (length - bufferStart)
         );
-        this.buffers[bufNr] = rafc.map(MapMode.READ_ONLY, bufferStart, bufSize);
+        this.buffers[bufNr] = rafc.map(MapMode.READ_ONLY, offset + bufferStart, bufSize);
         bufferStart += bufSize;
       }
       seek(0L);

Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/store/NIOFSDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/store/NIOFSDirectory.java?rev=1140809&r1=1140808&r2=1140809&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/store/NIOFSDirectory.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/store/NIOFSDirectory.java Tue Jun 28 19:42:31 2011
@@ -24,6 +24,9 @@ import java.nio.channels.ClosedChannelEx
 import java.nio.channels.FileChannel;
 import java.util.concurrent.Future; // javadoc
 
+import org.apache.lucene.store.SimpleFSDirectory.SimpleFSIndexInput;
+import org.apache.lucene.util.IOUtils;
+
 /**
  * An {@link FSDirectory} implementation that uses java.nio's FileChannel's
  * positional read, which allows multiple threads to read from the same file
@@ -77,6 +80,47 @@ public class NIOFSDirectory extends FSDi
     ensureOpen();
     return new NIOFSIndexInput(new File(getDirectory(), name), bufferSize, getReadChunkSize());
   }
+  
+  @Override
+  public CompoundFileDirectory openCompoundInput(String name, int bufferSize) throws IOException {
+    return new NIOFSCompoundFileDirectory(name, bufferSize);
+  }
+
+  private final class NIOFSCompoundFileDirectory extends CompoundFileDirectory {
+    private SimpleFSIndexInput.Descriptor fd;
+    private FileChannel fc;
+
+    public NIOFSCompoundFileDirectory(String fileName, int readBufferSize) throws IOException {
+      super(NIOFSDirectory.this, fileName, readBufferSize);
+      IndexInput stream = null;
+      try {
+        File f = new File(NIOFSDirectory.this.getDirectory(), fileName);
+        fd = new SimpleFSIndexInput.Descriptor(f, "r");
+        fc = fd.getChannel();
+        stream = new NIOFSIndexInput(fd, fc, 0, fd.length, readBufferSize,
+            getReadChunkSize());
+        initForRead(CompoundFileDirectory.readEntries(stream, NIOFSDirectory.this, fileName));
+        stream.close();
+      } catch (IOException e) {
+        // throw our original exception
+        IOUtils.closeSafely(e, fc, fd, stream);
+      }
+    }
+    
+    @Override
+    public IndexInput openInputSlice(String id, long offset, long length, int readBufferSize) throws IOException {
+      return new NIOFSIndexInput(fd, fc, offset, length, readBufferSize, getReadChunkSize());
+    }
+
+    @Override
+    public synchronized void close() throws IOException {
+      try {
+        IOUtils.closeSafely(false, fc, fd);
+      } finally {
+        super.close();
+      }
+    }
+  }
 
   protected static class NIOFSIndexInput extends SimpleFSDirectory.SimpleFSIndexInput {
 
@@ -91,6 +135,12 @@ public class NIOFSDirectory extends FSDi
       super(path, bufferSize, chunkSize);
       channel = file.getChannel();
     }
+    
+    public NIOFSIndexInput(Descriptor file, FileChannel fc, long off, long length, int bufferSize, int chunkSize) throws IOException {
+      super(file, off, length, bufferSize, chunkSize);
+      channel = fc;
+      isClone = true;
+    }
 
     @Override
     protected void newBuffer(byte[] newBuffer) {
@@ -145,7 +195,11 @@ public class NIOFSDirectory extends FSDi
       int readLength = bb.limit() - readOffset;
       assert readLength == len;
 
-      long pos = getFilePointer();
+      long pos = getFilePointer() + off;
+      
+      if (pos + len > end) {
+        throw new IOException("read past EOF");
+      }
 
       try {
         while (readLength > 0) {
@@ -159,9 +213,6 @@ public class NIOFSDirectory extends FSDi
           }
           bb.limit(limit);
           int i = channel.read(bb, pos);
-          if (i == -1) {
-            throw new IOException("read past EOF");
-          }
           pos += i;
           readOffset += i;
           readLength -= i;

Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/store/SimpleFSDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/store/SimpleFSDirectory.java?rev=1140809&r1=1140808&r2=1140809&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/store/SimpleFSDirectory.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/store/SimpleFSDirectory.java Tue Jun 28 19:42:31 2011
@@ -21,6 +21,8 @@ import java.io.File;
 import java.io.IOException;
 import java.io.RandomAccessFile;
 
+import org.apache.lucene.util.IOUtils;
+
 /** A straightforward implementation of {@link FSDirectory}
  *  using java.io.RandomAccessFile.  However, this class has
  *  poor concurrent performance (multiple threads will
@@ -55,6 +57,45 @@ public class SimpleFSDirectory extends F
     ensureOpen();
     return new SimpleFSIndexInput(new File(directory, name), bufferSize, getReadChunkSize());
   }
+  
+  @Override
+  public CompoundFileDirectory openCompoundInput(String name, int bufferSize) throws IOException {
+    return new SimpleFSCompoundFileDirectory(name, bufferSize);
+  }
+
+  private final class SimpleFSCompoundFileDirectory extends CompoundFileDirectory {
+    private SimpleFSIndexInput.Descriptor fd;
+
+    public SimpleFSCompoundFileDirectory(String fileName, int readBufferSize) throws IOException {
+      super(SimpleFSDirectory.this, fileName, readBufferSize);
+      IndexInput stream = null;
+      try {
+        final File f = new File(SimpleFSDirectory.this.getDirectory(), fileName);
+        fd = new SimpleFSIndexInput.Descriptor(f, "r");
+        stream = new SimpleFSIndexInput(fd, 0, fd.length, readBufferSize,
+            getReadChunkSize());
+        initForRead(CompoundFileDirectory.readEntries(stream, SimpleFSDirectory.this, fileName));
+        stream.close();
+      } catch (IOException e) {
+        // throw our original exception
+        IOUtils.closeSafely(e, fd, stream);
+      }
+    }
+
+    @Override
+    public IndexInput openInputSlice(String id, long offset, long length, int readBufferSize) throws IOException {
+      return new SimpleFSIndexInput(fd, offset, length, readBufferSize, getReadChunkSize());
+    }
+
+    @Override
+    public synchronized void close() throws IOException {
+      try {
+        fd.close();
+      } finally {
+        super.close();
+      }
+    }
+  }
 
   protected static class SimpleFSIndexInput extends BufferedIndexInput {
   
@@ -84,11 +125,24 @@ public class SimpleFSDirectory extends F
     boolean isClone;
     //  LUCENE-1566 - maximum read length on a 32bit JVM to prevent incorrect OOM 
     protected final int chunkSize;
+    protected final long off;
+    protected final long end;
     
     public SimpleFSIndexInput(File path, int bufferSize, int chunkSize) throws IOException {
       super(bufferSize);
-      file = new Descriptor(path, "r");
+      this.file = new Descriptor(path, "r"); 
       this.chunkSize = chunkSize;
+      this.off = 0L;
+      this.end = file.length;
+    }
+    
+    public SimpleFSIndexInput(Descriptor file, long off, long length, int bufferSize, int chunkSize) throws IOException {
+      super(bufferSize);
+      this.file = file;
+      this.chunkSize = chunkSize;
+      this.off = off;
+      this.end = off + length;
+      this.isClone = true; // well, we are sorta?
     }
   
     /** IndexInput methods */
@@ -96,13 +150,17 @@ public class SimpleFSDirectory extends F
     protected void readInternal(byte[] b, int offset, int len)
          throws IOException {
       synchronized (file) {
-        long position = getFilePointer();
+        long position = off + getFilePointer();
         if (position != file.position) {
           file.seek(position);
           file.position = position;
         }
         int total = 0;
 
+        if (position + len > end) {
+          throw new IOException("read past EOF");
+        }
+
         try {
           do {
             final int readLength;
@@ -113,9 +171,6 @@ public class SimpleFSDirectory extends F
               readLength = chunkSize;
             }
             final int i = file.read(b, offset + total, readLength);
-            if (i == -1) {
-              throw new IOException("read past EOF");
-            }
             file.position += i;
             total += i;
           } while (total < len);
@@ -144,7 +199,7 @@ public class SimpleFSDirectory extends F
   
     @Override
     public long length() {
-      return file.length;
+      return end - off;
     }
   
     @Override

Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/util/BitVector.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/util/BitVector.java?rev=1140809&r1=1140808&r2=1140809&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/util/BitVector.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/util/BitVector.java Tue Jun 28 19:42:31 2011
@@ -24,14 +24,16 @@ import org.apache.lucene.store.IndexInpu
 import org.apache.lucene.store.IndexOutput;
 
 /** Optimized implementation of a vector of bits.  This is more-or-less like
-  java.util.BitSet, but also includes the following:
-  <ul>
-  <li>a count() method, which efficiently computes the number of one bits;</li>
-  <li>optimized read from and write to disk;</li>
-  <li>inlinable get() method;</li>
-  <li>store and load, as bit set or d-gaps, depending on sparseness;</li> 
-  </ul>
-  */
+ *  java.util.BitSet, but also includes the following:
+ *  <ul>
+ *  <li>a count() method, which efficiently computes the number of one bits;</li>
+ *  <li>optimized read from and write to disk;</li>
+ *  <li>inlinable get() method;</li>
+ *  <li>store and load, as bit set or d-gaps, depending on sparseness;</li> 
+ *  </ul>
+ *
+ *  @lucene.internal
+ */
 public final class BitVector implements Cloneable, Bits {
 
   private byte[] bits;
@@ -41,16 +43,24 @@ public final class BitVector implements 
   /** Constructs a vector capable of holding <code>n</code> bits. */
   public BitVector(int n) {
     size = n;
-    bits = new byte[(size >> 3) + 1];
+    bits = new byte[getNumBytes(size)];
     count = 0;
   }
-  
+
   BitVector(byte[] bits, int size) {
     this.bits = bits;
     this.size = size;
     count = -1;
   }
   
+  private int getNumBytes(int size) {
+    int bytesLength = size >>> 3;
+    if ((size & 7) != 0) {
+      bytesLength++;
+    }
+    return bytesLength;
+  }
+  
   @Override
   public Object clone() {
     byte[] copyBits = new byte[bits.length];
@@ -158,6 +168,16 @@ public final class BitVector implements 
     4, 5, 5, 6, 5, 6, 6, 7, 5, 6, 6, 7, 6, 7, 7, 8
   };
 
+  private static String CODEC = "BitVector";
+
+  // Version before version tracking was added:
+  private final static int VERSION_PRE = -1;
+
+  // First version:
+  private final static int VERSION_START = 0;
+
+  // Increment version to change it:
+  private final static int VERSION_CURRENT = VERSION_START;
 
   /** Writes this vector to the file <code>name</code> in Directory
     <code>d</code>, in a format that can be read by the constructor {@link
@@ -165,6 +185,8 @@ public final class BitVector implements 
   public final void write(Directory d, String name) throws IOException {
     IndexOutput output = d.createOutput(name);
     try {
+      output.writeInt(-2);
+      CodecUtil.writeHeader(output, CODEC, VERSION_CURRENT);
       if (isSparse()) { 
         writeDgaps(output); // sparse bit-set more efficiently saved as d-gaps.
       } else {
@@ -202,19 +224,38 @@ public final class BitVector implements 
 
   /** Indicates if the bit vector is sparse and should be saved as a d-gaps list, or dense, and should be saved as a bit set. */
   private boolean isSparse() {
-    // note: order of comparisons below set to favor smaller values (no binary range search.)
-    // note: adding 4 because we start with ((int) -1) to indicate d-gaps format.
-    // note: we write the d-gap for the byte number, and the byte (bits[i]) itself, therefore
-    //       multiplying count by (8+8) or (8+16) or (8+24) etc.:
-    //       - first 8 for writing bits[i] (1 byte vs. 1 bit), and 
-    //       - second part for writing the byte-number d-gap as vint. 
+
+    final int setCount = count();
+    if (setCount == 0) {
+      return true;
+    }
+
+    final int avgGapLength = bits.length / setCount;
+
+    // expected number of bytes for vInt encoding of each gap
+    final int expectedDGapBytes;
+    if (avgGapLength <= (1<< 7)) {
+      expectedDGapBytes = 1;
+    } else if (avgGapLength <= (1<<14)) {
+      expectedDGapBytes = 2;
+    } else if (avgGapLength <= (1<<21)) {
+      expectedDGapBytes = 3;
+    } else if (avgGapLength <= (1<<28)) {
+      expectedDGapBytes = 4;
+    } else {
+      expectedDGapBytes = 5;
+    }
+
+    // +1 because we write the byte itself that contains the
+    // set bit
+    final int bytesPerSetBit = expectedDGapBytes + 1;
+    
+    // note: adding 32 because we start with ((int) -1) to indicate d-gaps format.
+    final long expectedBits = 32 + 8 * bytesPerSetBit * count();
+
     // note: factor is for read/write of byte-arrays being faster than vints.  
-    int factor = 10;  
-    if (bits.length < (1<< 7)) return factor * (4 + (8+ 8)*count()) < size();
-    if (bits.length < (1<<14)) return factor * (4 + (8+16)*count()) < size();
-    if (bits.length < (1<<21)) return factor * (4 + (8+24)*count()) < size();
-    if (bits.length < (1<<28)) return factor * (4 + (8+32)*count()) < size();
-    return                            factor * (4 + (8+40)*count()) < size();
+    final long factor = 10;  
+    return factor * expectedBits < size();
   }
 
   /** Constructs a bit vector from the file <code>name</code> in Directory
@@ -222,8 +263,18 @@ public final class BitVector implements 
     */
   public BitVector(Directory d, String name) throws IOException {
     IndexInput input = d.openInput(name);
+
     try {
-      size = input.readInt();       // read size
+      final int firstInt = input.readInt();
+      final int version;
+      if (firstInt == -2) {
+        // New format, with full header & version:
+        version = CodecUtil.checkHeader(input, CODEC, VERSION_START, VERSION_START);
+        size = input.readInt();
+      } else {
+        version = VERSION_PRE;
+        size = firstInt;
+      }
       if (size == -1) {
         readDgaps(input);
       } else {
@@ -237,7 +288,7 @@ public final class BitVector implements 
   /** Read as a bit set */
   private void readBits(IndexInput input) throws IOException {
     count = input.readInt();        // read count
-    bits = new byte[(size >> 3) + 1];     // allocate bits
+    bits = new byte[getNumBytes(size)];     // allocate bits
     input.readBytes(bits, 0, bits.length);
   }
 
@@ -254,30 +305,4 @@ public final class BitVector implements 
       n -= BYTE_COUNTS[bits[last] & 0xFF];
     }          
   }
-
-  /**
-   * Retrieve a subset of this BitVector.
-   * 
-   * @param start
-   *            starting index, inclusive
-   * @param end
-   *            ending index, exclusive
-   * @return subset
-   */
-  public BitVector subset(int start, int end) {
-    if (start < 0 || end > size() || end < start)
-      throw new IndexOutOfBoundsException();
-    // Special case -- return empty vector is start == end
-    if (end == start) return new BitVector(0);
-    byte[] bits = new byte[((end - start - 1) >>> 3) + 1];
-    int s = start >>> 3;
-    for (int i = 0; i < bits.length; i++) {
-      int cur = 0xFF & this.bits[i + s];
-      int next = i + s + 1 >= this.bits.length ? 0 : 0xFF & this.bits[i + s + 1];
-      bits[i] = (byte) ((cur >>> (start & 7)) | ((next << (8 - (start & 7)))));
-    }
-    int bitsToClear = (bits.length * 8 - (end - start)) % 8;
-    bits[bits.length - 1] &= ~(0xFF << (8 - bitsToClear));
-    return new BitVector(bits, end - start);
-  }
 }

Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/util/FieldCacheSanityChecker.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/util/FieldCacheSanityChecker.java?rev=1140809&r1=1140808&r2=1140809&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/util/FieldCacheSanityChecker.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/util/FieldCacheSanityChecker.java Tue Jun 28 19:42:31 2011
@@ -208,7 +208,7 @@ public final class FieldCacheSanityCheck
       
       if (seen.contains(rf)) continue;
 
-      List<Object> kids = getAllDecendentReaderKeys(rf.readerKey);
+      List<Object> kids = getAllDescendentReaderKeys(rf.readerKey);
       for (Object kidKey : kids) {
         ReaderField kid = new ReaderField(kidKey, rf.fieldName);
         
@@ -252,7 +252,7 @@ public final class FieldCacheSanityCheck
       badness = badEntries.toArray(badness);
 
       insanity.add(new Insanity(InsanityType.SUBREADER,
-                                "Found caches for decendents of " + 
+                                "Found caches for descendants of " + 
                                 parent.toString(),
                                 badness));
     }
@@ -266,7 +266,7 @@ public final class FieldCacheSanityCheck
    * the hierarchy of subReaders building up a list of the objects 
    * returned by obj.getFieldCacheKey()
    */
-  private List<Object> getAllDecendentReaderKeys(Object seed) {
+  private List<Object> getAllDescendentReaderKeys(Object seed) {
     List<Object> all = new ArrayList<Object>(17); // will grow as we iter
     all.add(seed);
     for (int i = 0; i < all.size(); i++) {

Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/util/OpenBitSet.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/util/OpenBitSet.java?rev=1140809&r1=1140808&r2=1140809&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/util/OpenBitSet.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/util/OpenBitSet.java Tue Jun 28 19:42:31 2011
@@ -664,15 +664,19 @@ public class OpenBitSet extends DocIdSet
    *  -1 is returned if there are no more set bits.
    */
   public int prevSetBit(int index) {
-    if (index < 0) {
-      return -1;
-    }
-    int i = index>>6;
+    int i = index >> 6;
+    final int subIndex;
+    long word;
     if (i >= wlen) {
       i = wlen - 1;
+      if (i < 0) return -1;
+      subIndex = 63;  // last possible bit
+      word = bits[i];
+    } else {
+      if (i < 0) return -1;
+      subIndex = index & 0x3f;  // index within the word
+      word = (bits[i] << (63-subIndex));  // skip all the bits to the left of index
     }
-    final int subIndex = index & 0x3f;      // index within the word
-    long word = (bits[i] << (63-subIndex));  // skip all the bits to the left of index
 
     if (word != 0) {
       return (i << 6) + subIndex - Long.numberOfLeadingZeros(word); // See LUCENE-3197

Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/util/PerReaderTermState.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/util/PerReaderTermState.java?rev=1140809&r1=1140808&r2=1140809&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/util/PerReaderTermState.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/util/PerReaderTermState.java Tue Jun 28 19:42:31 2011
@@ -28,7 +28,6 @@ import org.apache.lucene.index.Terms;
 import org.apache.lucene.index.TermsEnum;
 import org.apache.lucene.index.IndexReader.AtomicReaderContext;
 import org.apache.lucene.index.IndexReader.ReaderContext;
-import org.apache.lucene.index.TermsEnum.SeekStatus;
 
 /**
  * Maintains a {@link IndexReader} {@link TermState} view over
@@ -90,7 +89,7 @@ public final class PerReaderTermState {
         final Terms terms = fields.terms(field);
         if (terms != null) {
           final TermsEnum termsEnum = terms.getThreadTermsEnum(); // thread-private don't share!
-          if (SeekStatus.FOUND == termsEnum.seek(bytes, cache)) { 
+          if (termsEnum.seekExact(bytes, cache)) { 
             final TermState termState = termsEnum.termState();
             perReaderTermState.register(termState, leaves[i].ord, termsEnum.docFreq());
           }

Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/util/StringHelper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/util/StringHelper.java?rev=1140809&r1=1140808&r2=1140809&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/util/StringHelper.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/util/StringHelper.java Tue Jun 28 19:42:31 2011
@@ -27,17 +27,6 @@ import java.util.StringTokenizer;
  * @lucene.internal
  */
 public abstract class StringHelper {
-  /**
-   * Expert:
-   * The StringInterner implementation used by Lucene.
-   * This shouldn't be changed to an incompatible implementation after other Lucene APIs have been used.
-   */
-  public static StringInterner interner = new SimpleStringInterner(1024,8);
-
-  /** Return the same string object for all equal strings */
-  public static String intern(String s) {
-    return interner.intern(s);
-  }
 
   /**
    * Compares two byte[] arrays, element by element, and returns the
@@ -76,7 +65,7 @@ public abstract class StringHelper {
         if (bTokens.hasMoreTokens()) {
           int bToken = Integer.parseInt(bTokens.nextToken());
           if (aToken != bToken) {
-            return aToken - bToken;
+            return aToken < bToken ? -1 : 1;
           }
         } else {
           // a has some extra trailing tokens. if these are all zeroes, thats ok.
@@ -95,4 +84,12 @@ public abstract class StringHelper {
       return 0;
     }
   };
+
+  public static boolean equals(String s1, String s2) {
+    if (s1 == null) {
+      return s2 == null;
+    } else {
+      return s1.equals(s2);
+    }
+  }
 }

Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/util/Version.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/util/Version.java?rev=1140809&r1=1140808&r2=1140809&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/util/Version.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/util/Version.java Tue Jun 28 19:42:31 2011
@@ -58,6 +58,13 @@ public enum Version {
    */
   @Deprecated
   LUCENE_33,
+  
+  /**
+   * Match settings and bugs in Lucene's 3.4 release.
+   * @deprecated (4.0) Use latest
+   */
+  @Deprecated
+  LUCENE_34,
 
   /** Match settings and bugs in Lucene's 4.0 release. 
    *  <p>

Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/util/fst/BytesRefFSTEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/util/fst/BytesRefFSTEnum.java?rev=1140809&r1=1140808&r2=1140809&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/util/fst/BytesRefFSTEnum.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/util/fst/BytesRefFSTEnum.java Tue Jun 28 19:42:31 2011
@@ -71,6 +71,21 @@ public final class BytesRefFSTEnum<T> ex
     return setResult();
   }
 
+  /** Seeks to exactly this term, returning null if the term
+   *  doesn't exist.  This is faster than using {@link
+   *  #seekFloor} or {@link #seekCeil} because it
+   *  short-circuits as soon the match is not found. */
+  public InputOutput<T> seekExact(BytesRef target) throws IOException {
+    this.target = target;
+    targetLength = target.length;
+    if (super.doSeekExact()) {
+      assert upto == 1+target.length;
+      return setResult();
+    } else {
+      return null;
+    }
+  }
+
   @Override
   protected int getTargetLabel() {
     if (upto-1 == target.length) {

Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/util/fst/FST.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/util/fst/FST.java?rev=1140809&r1=1140808&r2=1140809&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/util/fst/FST.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/util/fst/FST.java Tue Jun 28 19:42:31 2011
@@ -704,6 +704,12 @@ public class FST<T> {
 
     if (labelToMatch == END_LABEL) {
       if (follow.isFinal()) {
+        if (follow.target <= 0) {
+          arc.flags = BIT_LAST_ARC;
+        } else {
+          arc.flags = 0;
+          arc.nextArc = follow.target;
+        }
         arc.output = follow.nextFinalOutput;
         arc.label = END_LABEL;
         return arc;

Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/util/fst/FSTEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/util/fst/FSTEnum.java?rev=1140809&r1=1140808&r2=1140809&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/util/fst/FSTEnum.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/util/fst/FSTEnum.java Tue Jun 28 19:42:31 2011
@@ -73,6 +73,7 @@ abstract class FSTEnum<T> {
       final int cmp = getCurrentLabel() - getTargetLabel();
       if (cmp < 0) {
         // seek forward
+        //System.out.println("    seek fwd");
         break;
       } else if (cmp > 0) {
         // seek backwards -- reset this arc to the first arc
@@ -83,6 +84,7 @@ abstract class FSTEnum<T> {
       }
       upto++;
     }
+    //System.out.println("  fall through upto=" + upto);
   }
 
   protected void doNext() throws IOException {
@@ -352,7 +354,7 @@ abstract class FSTEnum<T> {
           //System.out.println(" hasFloor arcIdx=" + (arc.arcIdx+1));
           fst.readNextRealArc(arc);
           assert arc.isLast() || fst.readNextArcLabel(arc) > targetLabel;
-          assert arc.label < targetLabel;
+          assert arc.label < targetLabel: "arc.label=" + arc.label + " vs targetLabel=" + targetLabel;
           pushLast();
           return;
         }        
@@ -410,6 +412,48 @@ abstract class FSTEnum<T> {
     }
   }
 
+  /** Seeks to exactly target term. */
+  protected boolean doSeekExact() throws IOException {
+
+    // TODO: possibly caller could/should provide common
+    // prefix length?  ie this work may be redundant if
+    // caller is in fact intersecting against its own
+    // automaton
+
+    //System.out.println("FE: seek exact upto=" + upto);
+
+    // Save time by starting at the end of the shared prefix
+    // b/w our current term & the target:
+    rewindPrefix();
+
+    //System.out.println("FE: after rewind upto=" + upto);
+    FST.Arc<T> arc = getArc(upto-1);
+    int targetLabel = getTargetLabel();
+
+    while(true) {
+      //System.out.println("  cycle target=" + (targetLabel == -1 ? "-1" : (char) targetLabel));
+      final FST.Arc<T> nextArc = fst.findTargetArc(targetLabel, arc, getArc(upto));
+      if (nextArc == null) {
+        // short circuit
+        //upto--;
+        //upto = 0;
+        fst.readFirstTargetArc(arc, getArc(upto));
+        //System.out.println("  no match upto=" + upto);
+        return false;
+      }
+      // Match -- recurse:
+      output[upto] = fst.outputs.add(output[upto-1], nextArc.output);
+      if (targetLabel == FST.END_LABEL) {
+        //System.out.println("  return found; upto=" + upto + " output=" + output[upto] + " nextArc=" + nextArc.isLast());
+        return true;
+      }
+      setCurrentLabel(targetLabel);
+      incr();
+      targetLabel = getTargetLabel();
+      arc = nextArc;
+    }
+  }
+
   private void incr() {
     upto++;
     grow();

Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/util/fst/IntsRefFSTEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/util/fst/IntsRefFSTEnum.java?rev=1140809&r1=1140808&r2=1140809&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/util/fst/IntsRefFSTEnum.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/util/fst/IntsRefFSTEnum.java Tue Jun 28 19:42:31 2011
@@ -71,6 +71,21 @@ public final class IntsRefFSTEnum<T> ext
     return setResult();
   }
 
+  /** Seeks to exactly this term, returning null if the term
+   *  doesn't exist.  This is faster than using {@link
+   *  #seekFloor} or {@link #seekCeil} because it
+   *  short-circuits as soon the match is not found. */
+  public InputOutput<T> seekExact(IntsRef target) throws IOException {
+    this.target = target;
+    targetLength = target.length;
+    if (super.doSeekExact()) {
+      assert upto == 1+target.length;
+      return setResult();
+    } else {
+      return null;
+    }
+  }
+
   @Override
   protected int getTargetLabel() {
     if (upto-1 == target.length) {

Modified: lucene/dev/branches/solr2452/lucene/src/site/build/site/fileformats.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/site/build/site/fileformats.html?rev=1140809&r1=1140808&r2=1140809&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/site/build/site/fileformats.html (original)
+++ lucene/dev/branches/solr2452/lucene/src/site/build/site/fileformats.html Tue Jun 28 19:42:31 2011
@@ -351,7 +351,7 @@ document.write("Last Published: " + docu
                 languages</a>.  If these versions are to remain compatible with Apache
                 Lucene, then a language-independent definition of the Lucene index
                 format is required.  This document thus attempts to provide a
-                complete and independent definition of the Apache Lucene 4.0 file
+                complete and independent definition of the Apache Lucene file
                 formats.
             </p>
 <p>
@@ -728,6 +728,14 @@ document.write("Last Published: " + docu
               that frequently run out of file handles.</td>
             
 </tr>
+              
+<tr>
+              
+<td><a href="#Compound File">Compound File Entry table</a></td>
+              <td>.cfe</td>
+              <td>The "virtual" compound file's entry table holding all entries in the corresponding .cfs file (Since 3.4)</td>
+            
+</tr>
             
 <tr>
               
@@ -832,10 +840,10 @@ document.write("Last Published: " + docu
 </div>
 
         
-<a name="N10204"></a><a name="Primitive Types"></a>
+<a name="N10212"></a><a name="Primitive Types"></a>
 <h2 class="boxed">Primitive Types</h2>
 <div class="section">
-<a name="N10209"></a><a name="Byte"></a>
+<a name="N10217"></a><a name="Byte"></a>
 <h3 class="boxed">Byte</h3>
 <p>
                     The most primitive type
@@ -843,7 +851,7 @@ document.write("Last Published: " + docu
                     other data types are defined as sequences
                     of bytes, so file formats are byte-order independent.
                 </p>
-<a name="N10212"></a><a name="UInt32"></a>
+<a name="N10220"></a><a name="UInt32"></a>
 <h3 class="boxed">UInt32</h3>
 <p>
                     32-bit unsigned integers are written as four
@@ -853,7 +861,7 @@ document.write("Last Published: " + docu
                     UInt32    --&gt; &lt;Byte&gt;<sup>4</sup>
                 
 </p>
-<a name="N10221"></a><a name="Uint64"></a>
+<a name="N1022F"></a><a name="Uint64"></a>
 <h3 class="boxed">Uint64</h3>
 <p>
                     64-bit unsigned integers are written as eight
@@ -862,7 +870,7 @@ document.write("Last Published: " + docu
 <p>UInt64    --&gt; &lt;Byte&gt;<sup>8</sup>
                 
 </p>
-<a name="N10230"></a><a name="VInt"></a>
+<a name="N1023E"></a><a name="VInt"></a>
 <h3 class="boxed">VInt</h3>
 <p>
                     A variable-length format for positive integers is
@@ -1412,13 +1420,13 @@ document.write("Last Published: " + docu
                     This provides compression while still being
                     efficient to decode.
                 </p>
-<a name="N10515"></a><a name="Chars"></a>
+<a name="N10523"></a><a name="Chars"></a>
 <h3 class="boxed">Chars</h3>
 <p>
                     Lucene writes unicode
                     character sequences as UTF-8 encoded bytes.
                 </p>
-<a name="N1051E"></a><a name="String"></a>
+<a name="N1052C"></a><a name="String"></a>
 <h3 class="boxed">String</h3>
 <p>
 		    Lucene writes strings as UTF-8 encoded bytes.
@@ -1431,10 +1439,10 @@ document.write("Last Published: " + docu
 </div>
 
         
-<a name="N1052B"></a><a name="Compound Types"></a>
+<a name="N10539"></a><a name="Compound Types"></a>
 <h2 class="boxed">Compound Types</h2>
 <div class="section">
-<a name="N10530"></a><a name="MapStringString"></a>
+<a name="N1053E"></a><a name="MapStringString"></a>
 <h3 class="boxed">Map&lt;String,String&gt;</h3>
 <p>
 		    In a couple places Lucene stores a Map
@@ -1447,13 +1455,13 @@ document.write("Last Published: " + docu
 </div>
 
         
-<a name="N10540"></a><a name="Per-Index Files"></a>
+<a name="N1054E"></a><a name="Per-Index Files"></a>
 <h2 class="boxed">Per-Index Files</h2>
 <div class="section">
 <p>
                 The files in this section exist one-per-index.
             </p>
-<a name="N10548"></a><a name="Segments File"></a>
+<a name="N10556"></a><a name="Segments File"></a>
 <h3 class="boxed">Segments File</h3>
 <p>
                     The active segments in the index are stored in the
@@ -1626,7 +1634,7 @@ document.write("Last Published: " + docu
 <p> HasVectors is 1 if this segment stores term vectors,
             else it's 0.
                 </p>
-<a name="N105D3"></a><a name="Lock File"></a>
+<a name="N105E1"></a><a name="Lock File"></a>
 <h3 class="boxed">Lock File</h3>
 <p>
                     The write lock, which is stored in the index
@@ -1640,27 +1648,29 @@ document.write("Last Published: " + docu
                     documents).  This lock file ensures that only one
                     writer is modifying the index at a time.
                 </p>
-<a name="N105DC"></a><a name="Deletable File"></a>
+<a name="N105EA"></a><a name="Deletable File"></a>
 <h3 class="boxed">Deletable File</h3>
 <p>
                     A writer dynamically computes
                     the files that are deletable, instead, so no file
                     is written.
                 </p>
-<a name="N105E5"></a><a name="Compound Files"></a>
+<a name="N105F3"></a><a name="Compound Files"></a>
 <h3 class="boxed">Compound Files</h3>
 <p>Starting with Lucene 1.4 the compound file format became default. This
                     is simply a container for all files described in the next section
 					(except for the .del file).</p>
-<p>Compound (.cfs) --&gt; FileCount, &lt;DataOffset, FileName&gt;
-                    <sup>FileCount</sup>
-                    ,
-                    FileData
+<p>Compound Entry Table (.cfe) --&gt; Version,  FileCount, &lt;FileName, DataOffset, DataLength&gt;
                     <sup>FileCount</sup>
                 
 </p>
+<p>Compound (.cfs) --&gt; FileData <sup>FileCount</sup>
+                
+</p>
+<p>Version --&gt; Int</p>
 <p>FileCount --&gt; VInt</p>
 <p>DataOffset --&gt; Long</p>
+<p>DataLength --&gt; Long</p>
 <p>FileName --&gt; String</p>
 <p>FileData --&gt; raw file data</p>
 <p>The raw file data is the data from the individual files named above.</p>
@@ -1674,14 +1684,14 @@ document.write("Last Published: " + docu
 </div>
 
         
-<a name="N1060D"></a><a name="Per-Segment Files"></a>
+<a name="N10624"></a><a name="Per-Segment Files"></a>
 <h2 class="boxed">Per-Segment Files</h2>
 <div class="section">
 <p>
                 The remaining files are all per-segment, and are
                 thus defined by suffix.
             </p>
-<a name="N10615"></a><a name="Fields"></a>
+<a name="N1062C"></a><a name="Fields"></a>
 <h3 class="boxed">Fields</h3>
 <p>
                     
@@ -1891,7 +1901,7 @@ document.write("Last Published: " + docu
 </li>
                 
 </ol>
-<a name="N106D0"></a><a name="Term Dictionary"></a>
+<a name="N106E7"></a><a name="Term Dictionary"></a>
 <h3 class="boxed">Term Dictionary</h3>
 <p>
                     The term dictionary is represented as two files:
@@ -2083,7 +2093,7 @@ document.write("Last Published: " + docu
 </li>
                 
 </ol>
-<a name="N10754"></a><a name="Frequencies"></a>
+<a name="N1076B"></a><a name="Frequencies"></a>
 <h3 class="boxed">Frequencies</h3>
 <p>
                     The .frq file contains the lists of documents
@@ -2211,7 +2221,7 @@ document.write("Last Published: " + docu
                    entry in level-1. In the example has entry 15 on level 1 a pointer to entry 15 on level 0 and entry 31 on level 1 a pointer
                    to entry 31 on level 0.                   
                 </p>
-<a name="N107DC"></a><a name="Positions"></a>
+<a name="N107F3"></a><a name="Positions"></a>
 <h3 class="boxed">Positions</h3>
 <p>
                     The .prx file contains the lists of positions that
@@ -2281,7 +2291,7 @@ document.write("Last Published: " + docu
                     Payload. If PayloadLength is not stored, then this Payload has the same
                     length as the Payload at the previous position.
                 </p>
-<a name="N10818"></a><a name="Normalization Factors"></a>
+<a name="N1082F"></a><a name="Normalization Factors"></a>
 <h3 class="boxed">Normalization Factors</h3>
 <p>There's a single .nrm file containing all norms:
                 </p>
@@ -2361,7 +2371,7 @@ document.write("Last Published: " + docu
                 </p>
 <p>Separate norm files are created (when adequate) for both compound and non compound segments.
                 </p>
-<a name="N10869"></a><a name="Term Vectors"></a>
+<a name="N10880"></a><a name="Term Vectors"></a>
 <h3 class="boxed">Term Vectors</h3>
 <p>
 		  Term Vector support is an optional on a field by
@@ -2497,7 +2507,7 @@ document.write("Last Published: " + docu
 </li>
                 
 </ol>
-<a name="N10905"></a><a name="Deleted Documents"></a>
+<a name="N1091C"></a><a name="Deleted Documents"></a>
 <h3 class="boxed">Deleted Documents</h3>
 <p>The .del file is
                     optional, and only exists when a segment contains deletions.
@@ -2561,7 +2571,7 @@ document.write("Last Published: " + docu
 </div>
 
         
-<a name="N1093F"></a><a name="Limitations"></a>
+<a name="N10956"></a><a name="Limitations"></a>
 <h2 class="boxed">Limitations</h2>
 <div class="section">
 <p>

Modified: lucene/dev/branches/solr2452/lucene/src/site/build/site/skin/images/rc-b-l-15-1body-2menu-3menu.png
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/site/build/site/skin/images/rc-b-l-15-1body-2menu-3menu.png?rev=1140809&r1=1140808&r2=1140809&view=diff
==============================================================================
Binary files - no diff available.

Modified: lucene/dev/branches/solr2452/lucene/src/site/build/site/skin/images/rc-b-r-15-1body-2menu-3menu.png
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/site/build/site/skin/images/rc-b-r-15-1body-2menu-3menu.png?rev=1140809&r1=1140808&r2=1140809&view=diff
==============================================================================
Binary files - no diff available.

Modified: lucene/dev/branches/solr2452/lucene/src/site/build/site/skin/images/rc-b-r-5-1header-2tab-selected-3tab-selected.png
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/site/build/site/skin/images/rc-b-r-5-1header-2tab-selected-3tab-selected.png?rev=1140809&r1=1140808&r2=1140809&view=diff
==============================================================================
Binary files - no diff available.

Modified: lucene/dev/branches/solr2452/lucene/src/site/build/site/skin/images/rc-t-l-5-1header-2searchbox-3searchbox.png
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/site/build/site/skin/images/rc-t-l-5-1header-2searchbox-3searchbox.png?rev=1140809&r1=1140808&r2=1140809&view=diff
==============================================================================
Binary files - no diff available.

Modified: lucene/dev/branches/solr2452/lucene/src/site/build/site/skin/images/rc-t-l-5-1header-2tab-selected-3tab-selected.png
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/site/build/site/skin/images/rc-t-l-5-1header-2tab-selected-3tab-selected.png?rev=1140809&r1=1140808&r2=1140809&view=diff
==============================================================================
Binary files - no diff available.

Modified: lucene/dev/branches/solr2452/lucene/src/site/build/site/skin/images/rc-t-l-5-1header-2tab-unselected-3tab-unselected.png
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/site/build/site/skin/images/rc-t-l-5-1header-2tab-unselected-3tab-unselected.png?rev=1140809&r1=1140808&r2=1140809&view=diff
==============================================================================
Binary files - no diff available.

Modified: lucene/dev/branches/solr2452/lucene/src/site/build/site/skin/images/rc-t-r-15-1body-2menu-3menu.png
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/site/build/site/skin/images/rc-t-r-15-1body-2menu-3menu.png?rev=1140809&r1=1140808&r2=1140809&view=diff
==============================================================================
Binary files - no diff available.

Modified: lucene/dev/branches/solr2452/lucene/src/site/build/site/skin/images/rc-t-r-5-1header-2searchbox-3searchbox.png
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/site/build/site/skin/images/rc-t-r-5-1header-2searchbox-3searchbox.png?rev=1140809&r1=1140808&r2=1140809&view=diff
==============================================================================
Binary files - no diff available.

Modified: lucene/dev/branches/solr2452/lucene/src/site/build/site/skin/images/rc-t-r-5-1header-2tab-selected-3tab-selected.png
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/site/build/site/skin/images/rc-t-r-5-1header-2tab-selected-3tab-selected.png?rev=1140809&r1=1140808&r2=1140809&view=diff
==============================================================================
Binary files - no diff available.

Modified: lucene/dev/branches/solr2452/lucene/src/site/build/site/skin/images/rc-t-r-5-1header-2tab-unselected-3tab-unselected.png
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/site/build/site/skin/images/rc-t-r-5-1header-2tab-unselected-3tab-unselected.png?rev=1140809&r1=1140808&r2=1140809&view=diff
==============================================================================
Binary files - no diff available.

Modified: lucene/dev/branches/solr2452/lucene/src/site/build/site/systemrequirements.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/site/build/site/systemrequirements.html?rev=1140809&r1=1140808&r2=1140809&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/site/build/site/systemrequirements.html (original)
+++ lucene/dev/branches/solr2452/lucene/src/site/build/site/systemrequirements.html Tue Jun 28 19:42:31 2011
@@ -240,10 +240,7 @@ document.write("Last Published: " + docu
         
 <li>ANT 1.7.0 or greater.</li>
         
-<li>JUnit 3.8.2 if you wish to run the unit tests.</li>
-        
-<li>CPU, Disk and Memory requirements are based on the many choices made in implementing Lucene (document size, number of documents, and number of hits retrieved to name a few.)
-            The benchmarks page has some information related to performance on particular platforms.</li>
+<li>CPU, Disk and Memory requirements are based on the many choices made in implementing Lucene (document size, number of documents, and number of hits retrieved to name a few.)</li>
     
 </ul>
 

Modified: lucene/dev/branches/solr2452/lucene/src/site/changes/changes2html.pl
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/site/changes/changes2html.pl?rev=1140809&r1=1140808&r2=1140809&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/site/changes/changes2html.pl (original)
+++ lucene/dev/branches/solr2452/lucene/src/site/changes/changes2html.pl Tue Jun 28 19:42:31 2011
@@ -23,7 +23,6 @@
 
 use strict;
 use warnings;
-use LWP::Simple;
 
 my $project_info_url = 'https://issues.apache.org/jira/rest/api/2.0.alpha1/project/LUCENE';
 my $jira_url_prefix = 'http://issues.apache.org/jira/browse/';
@@ -674,7 +673,9 @@ sub setup_release_dates {
            '2.4.0' => '2008-10-06',     '2.4.1' => '2009-03-09',
            '2.9.0' => '2009-09-23',     '2.9.1' => '2009-11-06',
            '3.0.0' => '2009-11-25');
-  my $project_info_json = get($project_info_url) || die "Unable to download $project_info_url";
+
+  my $project_info_json = get_url_contents($project_info_url);
+  
   my $project_info = json2perl($project_info_json);
   for my $version (@{$project_info->{versions}}) {
     if ($version->{releaseDate}) {
@@ -690,6 +691,21 @@ sub setup_release_dates {
   return %release_dates;
 }
 
+#
+# returns contents of the passed in url
+#
+sub get_url_contents {
+  my $url = shift;
+  my $tryWget = `wget --no-check-certificate -O - $url`;
+  if ($? eq 0) {
+    return $tryWget;
+  }
+  my $tryCurl = `curl $url`;
+  if ($? eq 0) {
+    return $tryCurl;
+  }
+  die "could not retrieve $url with either wget or curl!";
+}
 
 #
 # setup_month_regex

Modified: lucene/dev/branches/solr2452/lucene/src/site/src/documentation/content/xdocs/fileformats.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/site/src/documentation/content/xdocs/fileformats.xml?rev=1140809&r1=1140808&r2=1140809&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/site/src/documentation/content/xdocs/fileformats.xml (original)
+++ lucene/dev/branches/solr2452/lucene/src/site/src/documentation/content/xdocs/fileformats.xml Tue Jun 28 19:42:31 2011
@@ -27,7 +27,7 @@
                 languages</a>.  If these versions are to remain compatible with Apache
                 Lucene, then a language-independent definition of the Lucene index
                 format is required.  This document thus attempts to provide a
-                complete and independent definition of the Apache Lucene 4.0 file
+                complete and independent definition of the Apache Lucene file
                 formats.
             </p>
 
@@ -366,6 +366,11 @@
               <td>An optional "virtual" file consisting of all the other index files for systems
               that frequently run out of file handles.</td>
             </tr>
+              <tr>
+              <td><a href="#Compound File">Compound File Entry table</a></td>
+              <td>.cfe</td>
+              <td>The "virtual" compound file's entry table holding all entries in the corresponding .cfs file (Since 3.4)</td>
+            </tr>
             <tr>
               <td><a href="#Fields">Fields</a></td>
               <td>.fnm</td>
@@ -1129,17 +1134,20 @@
                 <p>Starting with Lucene 1.4 the compound file format became default. This
                     is simply a container for all files described in the next section
 					(except for the .del file).</p>
-
-                <p>Compound (.cfs) --&gt; FileCount, &lt;DataOffset, FileName&gt;
-                    <sup>FileCount</sup>
-                    ,
-                    FileData
+								<p>Compound Entry Table (.cfe) --&gt; Version,  FileCount, &lt;FileName, DataOffset, DataLength&gt;
                     <sup>FileCount</sup>
                 </p>
 
+                <p>Compound (.cfs) --&gt; FileData <sup>FileCount</sup>
+                </p>
+                
+								<p>Version --&gt; Int</p>
+								
                 <p>FileCount --&gt; VInt</p>
 
                 <p>DataOffset --&gt; Long</p>
+                
+                <p>DataLength --&gt; Long</p>
 
                 <p>FileName --&gt; String</p>
 



Mime
View raw message