lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From markrmil...@apache.org
Subject svn commit: r1181188 [3/8] - in /lucene/dev/branches/solrcloud: ./ dev-tools/eclipse/ dev-tools/idea/.idea/ dev-tools/idea/lucene/contrib/ dev-tools/idea/modules/grouping/ dev-tools/idea/solr/ dev-tools/idea/solr/contrib/analysis-extras/ dev-tools/idea...
Date Mon, 10 Oct 2011 19:58:32 GMT
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/values/Bytes.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/values/Bytes.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/values/Bytes.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/values/Bytes.java Mon Oct 10 19:58:24 2011
@@ -26,14 +26,12 @@ import java.util.concurrent.atomic.Atomi
 import org.apache.lucene.index.IndexFileNames;
 import org.apache.lucene.index.values.IndexDocValues.SortedSource;
 import org.apache.lucene.index.values.IndexDocValues.Source;
-import org.apache.lucene.index.values.IndexDocValues.SourceEnum;
 import org.apache.lucene.store.DataOutput;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.IOContext;
 import org.apache.lucene.store.IndexInput;
 import org.apache.lucene.store.IndexOutput;
 import org.apache.lucene.util.ArrayUtil;
-import org.apache.lucene.util.AttributeSource;
 import org.apache.lucene.util.ByteBlockPool;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.BytesRefHash;
@@ -50,8 +48,7 @@ import org.apache.lucene.util.packed.Pac
 /**
  * Provides concrete Writer/Reader implementations for <tt>byte[]</tt> value per
  * document. There are 6 package-private default implementations of this, for
- * all combinations of {@link Mode#DEREF}/{@link Mode#STRAIGHT}/
- * {@link Mode#SORTED} x fixed-length/variable-length.
+ * all combinations of {@link Mode#DEREF}/{@link Mode#STRAIGHT} x fixed-length/variable-length.
  * 
  * <p>
  * NOTE: Currently the total amount of byte[] data stored (across a single
@@ -101,11 +98,12 @@ public final class Bytes {
    *          the segment name and a unique id per segment.
    * @param mode
    *          the writers store mode
-   * @param comp
-   *          a {@link BytesRef} comparator - only used with {@link Mode#SORTED}
    * @param fixedSize
    *          <code>true</code> if all bytes subsequently passed to the
    *          {@link Writer} will have the same length
+   * @param sortComparator {@link BytesRef} comparator used by sorted variants. 
+   *        If <code>null</code> {@link BytesRef#getUTF8SortedAsUnicodeComparator()}
+   *        is used instead
    * @param bytesUsed
    *          an {@link AtomicLong} instance to track the used bytes within the
    *          {@link Writer}. A call to {@link Writer#finish(int)} will release
@@ -117,12 +115,12 @@ public final class Bytes {
    *           if the files for the writer can not be created.
    */
   public static Writer getWriter(Directory dir, String id, Mode mode,
-      Comparator<BytesRef> comp, boolean fixedSize, Counter bytesUsed, IOContext context)
+      boolean fixedSize, Comparator<BytesRef> sortComparator, Counter bytesUsed, IOContext context)
       throws IOException {
     // TODO -- i shouldn't have to specify fixed? can
     // track itself & do the write thing at write time?
-    if (comp == null) {
-      comp = BytesRef.getUTF8SortedAsUnicodeComparator();
+    if (sortComparator == null) {
+      sortComparator = BytesRef.getUTF8SortedAsUnicodeComparator();
     }
 
     if (fixedSize) {
@@ -131,7 +129,7 @@ public final class Bytes {
       } else if (mode == Mode.DEREF) {
         return new FixedDerefBytesImpl.Writer(dir, id, bytesUsed, context);
       } else if (mode == Mode.SORTED) {
-        return new FixedSortedBytesImpl.Writer(dir, id, comp, bytesUsed, context);
+        return new FixedSortedBytesImpl.Writer(dir, id, sortComparator, bytesUsed, context);
       }
     } else {
       if (mode == Mode.STRAIGHT) {
@@ -139,7 +137,7 @@ public final class Bytes {
       } else if (mode == Mode.DEREF) {
         return new VarDerefBytesImpl.Writer(dir, id, bytesUsed, context);
       } else if (mode == Mode.SORTED) {
-        return new VarSortedBytesImpl.Writer(dir, id, comp, bytesUsed, context);
+        return new VarSortedBytesImpl.Writer(dir, id, sortComparator, bytesUsed, context);
       }
     }
 
@@ -163,30 +161,34 @@ public final class Bytes {
    *          otherwise <code>false</code>
    * @param maxDoc
    *          the number of document values stored for the given ID
-   * @param sortComparator byte comparator used by sorted variants
+   * @param sortComparator {@link BytesRef} comparator used by sorted variants. 
+   *        If <code>null</code> {@link BytesRef#getUTF8SortedAsUnicodeComparator()}
+   *        is used instead
    * @return an initialized {@link IndexDocValues} instance.
    * @throws IOException
    *           if an {@link IOException} occurs
    */
   public static IndexDocValues getValues(Directory dir, String id, Mode mode,
       boolean fixedSize, int maxDoc, Comparator<BytesRef> sortComparator, IOContext context) throws IOException {
-
+    if (sortComparator == null) {
+      sortComparator = BytesRef.getUTF8SortedAsUnicodeComparator();
+    }
     // TODO -- I can peek @ header to determing fixed/mode?
     if (fixedSize) {
       if (mode == Mode.STRAIGHT) {
-        return new FixedStraightBytesImpl.Reader(dir, id, maxDoc, context);
+        return new FixedStraightBytesImpl.FixedStraightReader(dir, id, maxDoc, context);
       } else if (mode == Mode.DEREF) {
-        return new FixedDerefBytesImpl.Reader(dir, id, maxDoc, context);
+        return new FixedDerefBytesImpl.FixedDerefReader(dir, id, maxDoc, context);
       } else if (mode == Mode.SORTED) {
-        return new FixedSortedBytesImpl.Reader(dir, id, maxDoc, context);
+        return new FixedSortedBytesImpl.Reader(dir, id, maxDoc, context, ValueType.BYTES_FIXED_SORTED, sortComparator);
       }
     } else {
       if (mode == Mode.STRAIGHT) {
-        return new VarStraightBytesImpl.Reader(dir, id, maxDoc, context);
+        return new VarStraightBytesImpl.VarStraightReader(dir, id, maxDoc, context);
       } else if (mode == Mode.DEREF) {
-        return new VarDerefBytesImpl.Reader(dir, id, maxDoc, context);
+        return new VarDerefBytesImpl.VarDerefReader(dir, id, maxDoc, context);
       } else if (mode == Mode.SORTED) {
-        return new VarSortedBytesImpl.Reader(dir, id, maxDoc, sortComparator, context);
+        return new VarSortedBytesImpl.Reader(dir, id, maxDoc,context, ValueType.BYTES_VAR_SORTED, sortComparator);
       }
     }
 
@@ -196,7 +198,6 @@ public final class Bytes {
   // TODO open up this API?
   static abstract class BytesSourceBase extends Source {
     private final PagedBytes pagedBytes;
-    private final ValueType type;
     protected final IndexInput datIn;
     protected final IndexInput idxIn;
     protected final static int PAGED_BYTES_BITS = 15;
@@ -206,6 +207,7 @@ public final class Bytes {
 
     protected BytesSourceBase(IndexInput datIn, IndexInput idxIn,
         PagedBytes pagedBytes, long bytesToRead, ValueType type) throws IOException {
+      super(type);
       assert bytesToRead <= datIn.length() : " file size is less than the expected size diff: "
           + (bytesToRead - datIn.length()) + " pos: " + datIn.getFilePointer();
       this.datIn = datIn;
@@ -214,192 +216,15 @@ public final class Bytes {
       this.pagedBytes.copy(datIn, bytesToRead);
       data = pagedBytes.freeze(true);
       this.idxIn = idxIn;
-      this.type = type;
-    }
-
-    public void close() throws IOException {
-      try {
-        data.close(); // close data
-      } finally {
-        try {
-          if (datIn != null) {
-            datIn.close();
-          }
-        } finally {
-          if (idxIn != null) {// if straight - no index needed
-            idxIn.close();
-          }
-        }
-      }
-    }
-    
-    @Override
-    public ValueType type() {
-      return type;
-    }
-    
-
-    @Override
-    public int getValueCount() {
-      throw new UnsupportedOperationException();
     }
-
-    /**
-     * Returns one greater than the largest possible document number.
-     */
-    protected abstract int maxDoc();
-
-    @Override
-    public ValuesEnum getEnum(AttributeSource attrSource) throws IOException {
-      return new SourceEnum(attrSource, type(), this, maxDoc()) {
-        @Override
-        public int advance(int target) throws IOException {
-          if (target >= numDocs) {
-            return pos = NO_MORE_DOCS;
-          }
-          while (source.getBytes(target, bytesRef).length == 0) {
-            if (++target >= numDocs) {
-              return pos = NO_MORE_DOCS;
-            }
-          }
-          return pos = target;
-        }
-      };
-    }
-
   }
   
-  static abstract class DerefBytesSourceBase extends BytesSourceBase {
-    protected final PackedInts.Reader addresses;
-    public DerefBytesSourceBase(IndexInput datIn, IndexInput idxIn,  long bytesToRead, ValueType type) throws IOException {
-      super(datIn, idxIn, new PagedBytes(PAGED_BYTES_BITS), bytesToRead, type);
-      addresses = PackedInts.getReader(idxIn);
-    }
-    
-    @Override
-    public int getValueCount() {
-      return addresses.size();
-    }
-    
-    @Override
-    protected int maxDoc() {
-      return addresses.size();
-    }
-
-  }
-
-  static abstract class BytesSortedSourceBase extends SortedSource {
-    private final PagedBytes pagedBytes;
-    private final Comparator<BytesRef> comp;
-    protected final PackedInts.Reader docToOrdIndex;
-    private final ValueType type;
-    
-    protected final IndexInput datIn;
-    protected final IndexInput idxIn;
-    protected final BytesRef defaultValue = new BytesRef();
-    protected final static int PAGED_BYTES_BITS = 15;
-    protected final PagedBytes.Reader data;
-    
-
-    protected BytesSortedSourceBase(IndexInput datIn, IndexInput idxIn,
-        Comparator<BytesRef> comp, long bytesToRead, ValueType type) throws IOException {
-      this(datIn, idxIn, comp, new PagedBytes(PAGED_BYTES_BITS), bytesToRead, type);
-    }
-    
-    protected BytesSortedSourceBase(IndexInput datIn, IndexInput idxIn,
-        Comparator<BytesRef> comp, PagedBytes pagedBytes, long bytesToRead,ValueType type)
-        throws IOException {
-      assert bytesToRead <= datIn.length() : " file size is less than the expected size diff: "
-          + (bytesToRead - datIn.length()) + " pos: " + datIn.getFilePointer();
-      this.datIn = datIn;
-      this.pagedBytes = pagedBytes;
-      this.pagedBytes.copy(datIn, bytesToRead);
-      data = pagedBytes.freeze(true);
-      this.idxIn = idxIn;
-      this.comp = comp == null ? BytesRef.getUTF8SortedAsUnicodeComparator()
-          : comp;
-      docToOrdIndex = PackedInts.getReader(idxIn);
-      this.type = type;
-
-    }
-    
-    @Override
-    public int ord(int docID) {
-      return (int) docToOrdIndex.get(docID) -1;
-    }
-
-    @Override
-    public BytesRef getByOrd(int ord, BytesRef bytesRef) {
-      assert ord >= 0;
-      return deref(ord, bytesRef);
-    }
-
-    protected void closeIndexInput() throws IOException {
-      IOUtils.close(datIn, idxIn);
-    }
-    
-    /**
-     * Returns the largest doc id + 1 in this doc values source
-     */
-    public int maxDoc() {
-      return docToOrdIndex.size();
-    }
-    /**
-     * Copies the value for the given ord to the given {@link BytesRef} and
-     * returns it.
-     */
-    protected abstract BytesRef deref(int ord, BytesRef bytesRef);
-
-    protected int binarySearch(BytesRef b, BytesRef bytesRef, int low,
-        int high) {
-      int mid = 0;
-      while (low <= high) {
-        mid = (low + high) >>> 1;
-        deref(mid, bytesRef);
-        final int cmp = comp.compare(bytesRef, b);
-        if (cmp < 0) {
-          low = mid + 1;
-        } else if (cmp > 0) {
-          high = mid - 1;
-        } else {
-          return mid;
-        }
-      }
-      assert comp.compare(bytesRef, b) != 0;
-      return -(low + 1);
-    }
-
-    @Override
-    public ValuesEnum getEnum(AttributeSource attrSource) throws IOException {
-      return new SourceEnum(attrSource, type(), this, maxDoc()) {
-
-        @Override
-        public int advance(int target) throws IOException {
-          if (target >= numDocs) {
-            return pos = NO_MORE_DOCS;
-          }
-          while (source.getBytes(target, bytesRef).length == 0) {
-            if (++target >= numDocs) {
-              return pos = NO_MORE_DOCS;
-            }
-          }
-          return pos = target;
-        }
-      };
-    }
-    
-    @Override
-    public ValueType type() {
-      return type;
-    }
-  }
-
   // TODO: open up this API?!
   static abstract class BytesWriterBase extends Writer {
     private final String id;
     private IndexOutput idxOut;
     private IndexOutput datOut;
-    protected BytesRef bytesRef;
+    protected BytesRef bytesRef = new BytesRef();
     private final Directory dir;
     private final String codecName;
     private final int version;
@@ -467,8 +292,8 @@ public final class Bytes {
     public abstract void finish(int docCount) throws IOException;
 
     @Override
-    protected void mergeDoc(int docID) throws IOException {
-      add(docID, bytesRef);
+    protected void mergeDoc(int docID, int sourceDoc) throws IOException {
+      add(docID, currentMergeSource.getBytes(sourceDoc, bytesRef));
     }
 
     @Override
@@ -480,11 +305,6 @@ public final class Bytes {
     }
 
     @Override
-    protected void setNextEnum(ValuesEnum valuesEnum) {
-      bytesRef = valuesEnum.bytes();
-    }
-
-    @Override
     public void files(Collection<String> files) throws IOException {
       assert datOut != null;
       files.add(IndexFileNames.segmentFileName(id, "", DATA_EXTENSION));
@@ -506,30 +326,34 @@ public final class Bytes {
     protected final IndexInput datIn;
     protected final int version;
     protected final String id;
+    protected final ValueType type;
 
     protected BytesReaderBase(Directory dir, String id, String codecName,
-        int maxVersion, boolean doIndex, IOContext context) throws IOException {
-      this.id = id;
-      datIn = dir.openInput(IndexFileNames.segmentFileName(id, "",
-          Writer.DATA_EXTENSION), context);
+        int maxVersion, boolean doIndex, IOContext context, ValueType type) throws IOException {
+      IndexInput dataIn = null;
+      IndexInput indexIn = null;
       boolean success = false;
       try {
-        version = CodecUtil.checkHeader(datIn, codecName, maxVersion, maxVersion);
-        if (doIndex) {
-          idxIn = dir.openInput(IndexFileNames.segmentFileName(id, "",
-                                                               Writer.INDEX_EXTENSION), context);
-          final int version2 = CodecUtil.checkHeader(idxIn, codecName,
-                                                     maxVersion, maxVersion);
-          assert version == version2;
-        } else {
-          idxIn = null;
-        }
-        success = true;
+      dataIn = dir.openInput(IndexFileNames.segmentFileName(id, "",
+            Writer.DATA_EXTENSION), context);
+      version = CodecUtil.checkHeader(dataIn, codecName, maxVersion, maxVersion);
+      if (doIndex) {
+        indexIn = dir.openInput(IndexFileNames.segmentFileName(id, "",
+            Writer.INDEX_EXTENSION), context);
+        final int version2 = CodecUtil.checkHeader(indexIn, codecName,
+            maxVersion, maxVersion);
+        assert version == version2;
+      }
+      success = true;
       } finally {
         if (!success) {
-          closeInternal();
+          IOUtils.closeWhileHandlingException(dataIn, indexIn);
         }
       }
+      datIn = dataIn;
+      idxIn = indexIn;
+      this.type = type;
+      this.id = id;
     }
 
     /**
@@ -553,23 +377,20 @@ public final class Bytes {
       try {
         super.close();
       } finally {
-         closeInternal();
+         IOUtils.close(datIn, idxIn);
       }
     }
-    
-    private void closeInternal() throws IOException {
-      try {
-        datIn.close();
-      } finally {
-        if (idxIn != null) {
-          idxIn.close();
-        }
-      }
+
+    @Override
+    public ValueType type() {
+      return type;
     }
+    
   }
   
   static abstract class DerefBytesWriterBase extends BytesWriterBase {
     protected int size = -1;
+    protected int lastDocId = -1;
     protected int[] docToEntry;
     protected final BytesRefHash hash;
     
@@ -608,17 +429,33 @@ public final class Bytes {
         return;
       }
       checkSize(bytes);
+      fillDefault(docID);
       int ord = hash.add(bytes);
       if (ord < 0) {
         ord = (-ord) - 1;
       }
+      
+      docToEntry[docID] = ord;
+      lastDocId = docID;
+    }
+    
+    protected void fillDefault(int docID) {
       if (docID >= docToEntry.length) {
         final int size = docToEntry.length;
         docToEntry = ArrayUtil.grow(docToEntry, 1 + docID);
         bytesUsed.addAndGet((docToEntry.length - size)
             * RamUsageEstimator.NUM_BYTES_INT);
       }
-      docToEntry[docID] = 1 + ord;
+      assert size >= 0;
+      BytesRef ref = new BytesRef(size);
+      ref.length = size;
+      int ord = hash.add(ref);
+      if (ord < 0) {
+        ord = (-ord) - 1;
+      }
+      for (int i = lastDocId+1; i < docID; i++) {
+        docToEntry[i] = ord;
+      }
     }
     
     protected void checkSize(BytesRef bytes) {
@@ -713,77 +550,50 @@ public final class Bytes {
     
   }
   
-  abstract static class DerefBytesEnumBase extends ValuesEnum {
-    private final PackedInts.ReaderIterator idx;
-    private final int valueCount;
-    private int pos = -1;
+  static abstract class BytesSortedSourceBase extends SortedSource {
+    private final PagedBytes pagedBytes;
+    
+    protected final PackedInts.Reader docToOrdIndex;
     protected final IndexInput datIn;
-    protected final long fp;
-    protected final int size;
+    protected final IndexInput idxIn;
+    protected final BytesRef defaultValue = new BytesRef();
+    protected final static int PAGED_BYTES_BITS = 15;
+    protected final PagedBytes.Reader data;
 
-    protected DerefBytesEnumBase(AttributeSource source, IndexInput datIn,
-        IndexInput idxIn, int size, ValueType enumType) throws IOException {
-      super(source, enumType);
-      this.datIn = datIn;
-      this.size = size;
-      idx = PackedInts.getReaderIterator(idxIn);
-      fp = datIn.getFilePointer();
-      if (size > 0) {
-        bytesRef.grow(this.size);
-        bytesRef.length = this.size;
-      }
-      bytesRef.offset = 0;
-      valueCount = idx.size();
-    }
-
-    protected void copyFrom(ValuesEnum valuesEnum) {
-      bytesRef = valuesEnum.bytesRef;
-      if (bytesRef.bytes.length < size) {
-        bytesRef.grow(size);
-      }
-      bytesRef.length = size;
-      bytesRef.offset = 0;
+    protected BytesSortedSourceBase(IndexInput datIn, IndexInput idxIn,
+        Comparator<BytesRef> comp, long bytesToRead, ValueType type) throws IOException {
+      this(datIn, idxIn, comp, new PagedBytes(PAGED_BYTES_BITS), bytesToRead, type);
     }
+    
+    protected BytesSortedSourceBase(IndexInput datIn, IndexInput idxIn,
+        Comparator<BytesRef> comp, PagedBytes pagedBytes, long bytesToRead,ValueType type)
+        throws IOException {
+      super(type, comp);
+      assert bytesToRead <= datIn.length() : " file size is less than the expected size diff: "
+          + (bytesToRead - datIn.length()) + " pos: " + datIn.getFilePointer();
+      this.datIn = datIn;
+      this.pagedBytes = pagedBytes;
+      this.pagedBytes.copy(datIn, bytesToRead);
+      data = pagedBytes.freeze(true);
+      this.idxIn = idxIn;
+      docToOrdIndex = PackedInts.getReader(idxIn);
 
-    @Override
-    public int advance(int target) throws IOException {
-      if (target < valueCount) {
-        long address;
-        while ((address = idx.advance(target)) == 0) {
-          if (++target >= valueCount) {
-            return pos = NO_MORE_DOCS;
-          }
-        }
-        pos = idx.ord();
-        fill(address, bytesRef);
-        return pos;
-      }
-      return pos = NO_MORE_DOCS;
     }
-
+    
     @Override
-    public int nextDoc() throws IOException {
-      if (pos >= valueCount) {
-        return pos = NO_MORE_DOCS;
-      }
-      return advance(pos + 1);
+    public int ord(int docID) {
+      return (int) docToOrdIndex.get(docID);
     }
 
-    public void close() throws IOException {
-      try {
-        datIn.close();
-      } finally {
-        idx.close();
-      }
+    protected void closeIndexInput() throws IOException {
+      IOUtils.close(datIn, idxIn);
     }
-
-    protected abstract void fill(long address, BytesRef ref) throws IOException;
-
-    @Override
-    public int docID() {
-      return pos;
+    
+    /**
+     * Returns the largest doc id + 1 in this doc values source
+     */
+    public int maxDoc() {
+      return docToOrdIndex.size();
     }
-
   }
-
-}
\ No newline at end of file
+}

Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/values/FixedDerefBytesImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/values/FixedDerefBytesImpl.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/values/FixedDerefBytesImpl.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/values/FixedDerefBytesImpl.java Mon Oct 10 19:58:24 2011
@@ -20,16 +20,17 @@ package org.apache.lucene.index.values;
 import java.io.IOException;
 
 import org.apache.lucene.index.values.Bytes.BytesReaderBase;
-import org.apache.lucene.index.values.Bytes.DerefBytesSourceBase;
-import org.apache.lucene.index.values.Bytes.DerefBytesEnumBase;
+import org.apache.lucene.index.values.Bytes.BytesSourceBase;
 import org.apache.lucene.index.values.Bytes.DerefBytesWriterBase;
+import org.apache.lucene.index.values.DirectSource;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.IOContext;
 import org.apache.lucene.store.IndexInput;
 import org.apache.lucene.store.IndexOutput;
-import org.apache.lucene.util.AttributeSource;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.Counter;
+import org.apache.lucene.util.PagedBytes;
+import org.apache.lucene.util.packed.PackedInts;
 
 // Stores fixed-length byte[] by deref, ie when two docs
 // have the same value, they store only 1 byte[]
@@ -66,63 +67,67 @@ class FixedDerefBytesImpl {
     }
   }
 
-  public static class Reader extends BytesReaderBase {
+  public static class FixedDerefReader extends BytesReaderBase {
     private final int size;
     private final int numValuesStored;
-    Reader(Directory dir, String id, int maxDoc, IOContext context) throws IOException {
-      super(dir, id, CODEC_NAME, VERSION_START, true, context);
+    FixedDerefReader(Directory dir, String id, int maxDoc, IOContext context) throws IOException {
+      super(dir, id, CODEC_NAME, VERSION_START, true, context, ValueType.BYTES_FIXED_DEREF);
       size = datIn.readInt();
       numValuesStored = idxIn.readInt();
     }
 
     @Override
     public Source load() throws IOException {
-      return new Source(cloneData(), cloneIndex(), size, numValuesStored);
+      return new FixedDerefSource(cloneData(), cloneIndex(), size, numValuesStored);
     }
 
-    private static final class Source extends DerefBytesSourceBase {
-      private final int size;
-
-      protected Source(IndexInput datIn, IndexInput idxIn, int size, long numValues) throws IOException {
-        super(datIn, idxIn, size * numValues, ValueType.BYTES_FIXED_DEREF);
-        this.size = size;
-      }
+    @Override
+    public Source getDirectSource()
+        throws IOException {
+      return new DirectFixedDerefSource(cloneData(), cloneIndex(), size, type());
+    }
 
-      @Override
-      public BytesRef getBytes(int docID, BytesRef bytesRef) {
-        final int id = (int) addresses.get(docID);
-        if (id == 0) {
-          bytesRef.length = 0;
-          return bytesRef;
-        }
-        return data.fillSlice(bytesRef, ((id - 1) * size), size);
-      }
+    @Override
+    public int getValueSize() {
+      return size;
+    }
+    
+  }
+  
+  static final class FixedDerefSource extends BytesSourceBase {
+    private final int size;
+    private final PackedInts.Reader addresses;
 
+    protected FixedDerefSource(IndexInput datIn, IndexInput idxIn, int size, long numValues) throws IOException {
+      super(datIn, idxIn, new PagedBytes(PAGED_BYTES_BITS), size * numValues,
+          ValueType.BYTES_FIXED_DEREF);
+      this.size = size;
+      addresses = PackedInts.getReader(idxIn);
     }
 
     @Override
-    public ValuesEnum getEnum(AttributeSource source) throws IOException {
-        return new DerefBytesEnum(source, cloneData(), cloneIndex(), size);
+    public BytesRef getBytes(int docID, BytesRef bytesRef) {
+      final int id = (int) addresses.get(docID);
+      return data.fillSlice(bytesRef, (id * size), size);
     }
 
-    final static class DerefBytesEnum extends DerefBytesEnumBase {
-
-      public DerefBytesEnum(AttributeSource source, IndexInput datIn,
-          IndexInput idxIn, int size) throws IOException {
-        super(source, datIn, idxIn, size, ValueType.BYTES_FIXED_DEREF);
-      }
+  }
+  
+  final static class DirectFixedDerefSource extends DirectSource {
+    private final PackedInts.RandomAccessReaderIterator index;
+    private final int size;
 
-      protected void fill(long address, BytesRef ref) throws IOException {
-        datIn.seek(fp + ((address - 1) * size));
-        datIn.readBytes(ref.bytes, 0, size);
-        ref.length = size;
-        ref.offset = 0;
-      }
+    DirectFixedDerefSource(IndexInput data, IndexInput index, int size, ValueType type)
+        throws IOException {
+      super(data, type);
+      this.size = size;
+      this.index = PackedInts.getRandomAccessReaderIterator(index);
     }
 
     @Override
-    public ValueType type() {
-      return ValueType.BYTES_FIXED_DEREF;
+    protected int position(int docID) throws IOException {
+      data.seek(baseOffset + index.get(docID) * size);
+      return size;
     }
   }
 

Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/values/FixedSortedBytesImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/values/FixedSortedBytesImpl.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/values/FixedSortedBytesImpl.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/values/FixedSortedBytesImpl.java Mon Oct 10 19:58:24 2011
@@ -23,14 +23,14 @@ import java.util.Comparator;
 import org.apache.lucene.index.values.Bytes.BytesSortedSourceBase;
 import org.apache.lucene.index.values.Bytes.BytesReaderBase;
 import org.apache.lucene.index.values.Bytes.DerefBytesWriterBase;
-import org.apache.lucene.index.values.FixedDerefBytesImpl.Reader.DerefBytesEnum;
+import org.apache.lucene.index.values.IndexDocValues.SortedSource;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.IOContext;
 import org.apache.lucene.store.IndexInput;
 import org.apache.lucene.store.IndexOutput;
-import org.apache.lucene.util.AttributeSource;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.Counter;
+import org.apache.lucene.util.packed.PackedInts;
 
 // Stores fixed-length byte[] by deref, ie when two docs
 // have the same value, they store only 1 byte[]
@@ -44,7 +44,7 @@ class FixedSortedBytesImpl {
   static final int VERSION_START = 0;
   static final int VERSION_CURRENT = VERSION_START;
 
-  static class Writer extends DerefBytesWriterBase {
+  static final class Writer extends DerefBytesWriterBase {
     private final Comparator<BytesRef> comp;
 
     public Writer(Directory dir, String id, Comparator<BytesRef> comp,
@@ -57,9 +57,10 @@ class FixedSortedBytesImpl {
     // some last docs that we didn't see
     @Override
     public void finishInternal(int docCount) throws IOException {
+      fillDefault(docCount);
       final IndexOutput datOut = getOrCreateDataOut();
       final int count = hash.size();
-      final int[] address = new int[count+1]; // addr 0 is default values
+      final int[] address = new int[count]; // addr 0 is default values
       datOut.writeInt(size);
       if (size != -1) {
         final int[] sortedEntries = hash.sort(comp);
@@ -70,7 +71,7 @@ class FixedSortedBytesImpl {
           final BytesRef bytes = hash.get(e, bytesRef);
           assert bytes.length == size;
           datOut.writeBytes(bytes.bytes, bytes.offset, bytes.length);
-          address[e + 1] = 1 + i;
+          address[e] = i;
         }
       }
       final IndexOutput idxOut = getOrCreateIndexOut();
@@ -79,65 +80,106 @@ class FixedSortedBytesImpl {
     }
   }
 
-  public static class Reader extends BytesReaderBase {
+  static final class Reader extends BytesReaderBase {
     private final int size;
-    private final int numValuesStored;
+    private final int valueCount;
+    private final Comparator<BytesRef> comparator;
 
-    public Reader(Directory dir, String id, int maxDoc, IOContext context) throws IOException {
-      super(dir, id, CODEC_NAME, VERSION_START, true, context);
+    public Reader(Directory dir, String id, int maxDoc, IOContext context,
+        ValueType type, Comparator<BytesRef> comparator) throws IOException {
+      super(dir, id, CODEC_NAME, VERSION_START, true, context, type);
       size = datIn.readInt();
-      numValuesStored = idxIn.readInt();
+      valueCount = idxIn.readInt();
+      this.comparator = comparator;
     }
 
     @Override
-    public org.apache.lucene.index.values.IndexDocValues.Source load()
-        throws IOException {
-      return loadSorted(null);
+    public Source load() throws IOException {
+      return new FixedSortedSource(cloneData(), cloneIndex(), size,
+          valueCount, comparator);
     }
 
     @Override
-    public SortedSource loadSorted(Comparator<BytesRef> comp)
-        throws IOException {
-      return new Source(cloneData(), cloneIndex(), size, numValuesStored, comp);
+    public Source getDirectSource() throws IOException {
+      return new DirectFixedSortedSource(cloneData(), cloneIndex(), size,
+          valueCount, comparator, type);
+    }
+    
+    @Override
+    public int getValueSize() {
+      return size;
     }
+  }
 
-    private static class Source extends BytesSortedSourceBase {
-      private final int valueCount;
-      private final int size;
-
-      public Source(IndexInput datIn, IndexInput idxIn, int size,
-          int numValues, Comparator<BytesRef> comp) throws IOException {
-        super(datIn, idxIn, comp, size * numValues, ValueType.BYTES_FIXED_SORTED);
-        this.size = size;
-        this.valueCount = numValues;
-        closeIndexInput();
-      }
+  static final class FixedSortedSource extends BytesSortedSourceBase {
+    private final int valueCount;
+    private final int size;
 
-      @Override
-      public int getByValue(BytesRef bytes, BytesRef tmpRef) {
-        return binarySearch(bytes, tmpRef, 0, valueCount - 1);
-      }
+    FixedSortedSource(IndexInput datIn, IndexInput idxIn, int size,
+        int numValues, Comparator<BytesRef> comp) throws IOException {
+      super(datIn, idxIn, comp, size * numValues, ValueType.BYTES_FIXED_SORTED);
+      this.size = size;
+      this.valueCount = numValues;
+      closeIndexInput();
+    }
 
-      @Override
-      public int getValueCount() {
-        return valueCount;
-      }
+    @Override
+    public int getValueCount() {
+      return valueCount;
+    }
 
-      @Override
-      protected BytesRef deref(int ord, BytesRef bytesRef) {
-        return data.fillSlice(bytesRef, (ord * size), size);
+    @Override
+    public BytesRef getByOrd(int ord, BytesRef bytesRef) {
+      return data.fillSlice(bytesRef, (ord * size), size);
+    }
+  }
+
+  static final class DirectFixedSortedSource extends SortedSource {
+    final PackedInts.RandomAccessReaderIterator docToOrdIndex;
+    private final IndexInput datIn;
+    private final long basePointer;
+    private final int size;
+    private final int valueCount;
+
+    DirectFixedSortedSource(IndexInput datIn, IndexInput idxIn, int size,
+        int valueCount, Comparator<BytesRef> comp, ValueType type)
+        throws IOException {
+      super(type, comp);
+      docToOrdIndex = PackedInts.getRandomAccessReaderIterator(idxIn);
+      basePointer = datIn.getFilePointer();
+      this.datIn = datIn;
+      this.size = size;
+      this.valueCount = valueCount;
+    }
+
+    @Override
+    public int ord(int docID) {
+      try {
+        return (int) docToOrdIndex.get(docID);
+      } catch (IOException e) {
+        throw new IllegalStateException("failed to get ord", e);
       }
     }
 
     @Override
-    public ValuesEnum getEnum(AttributeSource source) throws IOException {
-      // do unsorted
-      return new DerefBytesEnum(source, cloneData(), cloneIndex(), size);
+    public BytesRef getByOrd(int ord, BytesRef bytesRef) {
+      try {
+        datIn.seek(basePointer + size * ord);
+        if (bytesRef.bytes.length < size) {
+          bytesRef.grow(size);
+        }
+        datIn.readBytes(bytesRef.bytes, 0, size);
+        bytesRef.length = size;
+        bytesRef.offset = 0;
+        return bytesRef;
+      } catch (IOException ex) {
+        throw new IllegalStateException("failed to getByOrd", ex);
+      }
     }
 
     @Override
-    public ValueType type() {
-      return ValueType.BYTES_FIXED_SORTED;
+    public int getValueCount() {
+      return valueCount;
     }
   }
 }

Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/values/FixedStraightBytesImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/values/FixedStraightBytesImpl.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/values/FixedStraightBytesImpl.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/values/FixedStraightBytesImpl.java Mon Oct 10 19:58:24 2011
@@ -24,11 +24,12 @@ import java.io.IOException;
 import org.apache.lucene.index.values.Bytes.BytesSourceBase;
 import org.apache.lucene.index.values.Bytes.BytesReaderBase;
 import org.apache.lucene.index.values.Bytes.BytesWriterBase;
+import org.apache.lucene.index.values.DirectSource;
+import org.apache.lucene.index.values.IndexDocValues.Source;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.IOContext;
 import org.apache.lucene.store.IndexInput;
 import org.apache.lucene.store.IndexOutput;
-import org.apache.lucene.util.AttributeSource;
 import org.apache.lucene.util.ByteBlockPool;
 import org.apache.lucene.util.ByteBlockPool.DirectTrackingAllocator;
 import org.apache.lucene.util.BytesRef;
@@ -58,6 +59,7 @@ class FixedStraightBytesImpl {
         int version, Counter bytesUsed, IOContext context) throws IOException {
       super(dir, id, codecName, version, bytesUsed, context);
       pool = new ByteBlockPool(new DirectTrackingAllocator(bytesUsed));
+      pool.nextBuffer();
     }
     
     @Override
@@ -69,7 +71,6 @@ class FixedStraightBytesImpl {
           throw new IllegalArgumentException("bytes arrays > " + Short.MAX_VALUE + " are not supported");
         }
         size = bytes.length;
-        pool.nextBuffer();
       } else if (bytes.length != size) {
         throw new IllegalArgumentException("expected bytes size=" + size
             + " but got " + bytes.length);
@@ -119,7 +120,7 @@ class FixedStraightBytesImpl {
   }
 
   static class Writer extends FixedBytesWriterBase {
-    private boolean merge;
+    private boolean hasMerged;
     private IndexOutput datOut;
     
     public Writer(Directory dir, String id, Counter bytesUsed, IOContext context) throws IOException {
@@ -132,13 +133,16 @@ class FixedStraightBytesImpl {
 
 
     @Override
-    protected void merge(MergeState state) throws IOException {
-      merge = true;
+    protected void merge(SingleSubMergeState state) throws IOException {
       datOut = getOrCreateDataOut();
       boolean success = false;
       try {
-        if (state.liveDocs == null && state.reader instanceof Reader ) {
-          Reader reader = (Reader) state.reader;
+        if (!hasMerged && size != -1) {
+          datOut.writeInt(size);
+        }
+
+        if (state.liveDocs == null && tryBulkMerge(state.reader)) {
+          FixedStraightReader reader = (FixedStraightReader) state.reader;
           final int maxDocs = reader.maxDoc;
           if (maxDocs == 0) {
             return;
@@ -171,23 +175,33 @@ class FixedStraightBytesImpl {
         if (!success) {
           IOUtils.closeWhileHandlingException(datOut);
         }
+        hasMerged = true;
       }
     }
     
+    protected boolean tryBulkMerge(IndexDocValues docValues) {
+      return docValues instanceof FixedStraightReader;
+    }
+    
     @Override
-    protected void mergeDoc(int docID) throws IOException {
+    protected void mergeDoc(int docID, int sourceDoc) throws IOException {
       assert lastDocID < docID;
+      setMergeBytes(sourceDoc);
       if (size == -1) {
         size = bytesRef.length;
         datOut.writeInt(size);
       }
-      assert size == bytesRef.length;
+      assert size == bytesRef.length : "size: " + size + " ref: " + bytesRef.length;
       if (lastDocID+1 < docID) {
         fill(datOut, docID);
       }
       datOut.writeBytes(bytesRef.bytes, bytesRef.offset, bytesRef.length);
       lastDocID = docID;
     }
+    
+    protected void setMergeBytes(int sourceDoc) {
+      currentMergeSource.getBytes(sourceDoc, bytesRef);
+    }
 
 
 
@@ -201,7 +215,7 @@ class FixedStraightBytesImpl {
     public void finish(int docCount) throws IOException {
       boolean success = false;
       try {
-        if (!merge) {
+        if (!hasMerged) {
           // indexing path - no disk IO until here
           assert datOut == null;
           datOut = getOrCreateDataOut();
@@ -236,16 +250,16 @@ class FixedStraightBytesImpl {
   
   }
   
-  public static class Reader extends BytesReaderBase {
+  public static class FixedStraightReader extends BytesReaderBase {
     protected final int size;
     protected final int maxDoc;
     
-    Reader(Directory dir, String id, int maxDoc, IOContext context) throws IOException {
-      this(dir, id, CODEC_NAME, VERSION_CURRENT, maxDoc, context);
+    FixedStraightReader(Directory dir, String id, int maxDoc, IOContext context) throws IOException {
+      this(dir, id, CODEC_NAME, VERSION_CURRENT, maxDoc, context, ValueType.BYTES_FIXED_STRAIGHT);
     }
 
-    protected Reader(Directory dir, String id, String codec, int version, int maxDoc, IOContext context) throws IOException {
-      super(dir, id, codec, version, false, context);
+    protected FixedStraightReader(Directory dir, String id, String codec, int version, int maxDoc, IOContext context, ValueType type) throws IOException {
+      super(dir, id, codec, version, false, context, type);
       size = datIn.readInt();
       this.maxDoc = maxDoc;
     }
@@ -253,155 +267,88 @@ class FixedStraightBytesImpl {
     @Override
     public Source load() throws IOException {
       return size == 1 ? new SingleByteSource(cloneData(), maxDoc) : 
-        new StraightBytesSource(cloneData(), size, maxDoc);
+        new FixedStraightSource(cloneData(), size, maxDoc, type);
     }
 
     @Override
     public void close() throws IOException {
       datIn.close();
     }
+   
+    @Override
+    public Source getDirectSource() throws IOException {
+      return new DirectFixedStraightSource(cloneData(), size, type());
+    }
     
-    // specialized version for single bytes
-    private static class SingleByteSource extends Source {
-      private final int maxDoc;
-      private final byte[] data;
-
-      public SingleByteSource(IndexInput datIn, int maxDoc) throws IOException {
-        this.maxDoc = maxDoc;
-        try {
-          data = new byte[maxDoc];
-          datIn.readBytes(data, 0, data.length, false);
-        } finally {
-          IOUtils.close(datIn);
-        }
-
-      }
-
-      @Override
-      public BytesRef getBytes(int docID, BytesRef bytesRef) {
-        bytesRef.length = 1;
-        bytesRef.bytes = data;
-        bytesRef.offset = docID;
-        return bytesRef;
-      }
-      
-      @Override
-      public ValueType type() {
-        return ValueType.BYTES_FIXED_STRAIGHT;
-      }
-
-      @Override
-      public ValuesEnum getEnum(AttributeSource attrSource) throws IOException {
-        return new SourceEnum(attrSource, type(), this, maxDoc) {
-          @Override
-          public int advance(int target) throws IOException {
-            if (target >= numDocs) {
-              return pos = NO_MORE_DOCS;
-            }
-            bytesRef.length = 1;
-            bytesRef.bytes = data;
-            bytesRef.offset = target;
-            return pos = target;
-          }
-        };
-      }
-
+    @Override
+    public int getValueSize() {
+      return size;
     }
+  }
+  
+  // specialized version for single bytes
+  private static final class SingleByteSource extends Source {
+    private final byte[] data;
 
-    private final static class StraightBytesSource extends BytesSourceBase {
-      private final int size;
-      private final int maxDoc;
-
-      public StraightBytesSource(IndexInput datIn, int size, int maxDoc)
-          throws IOException {
-        super(datIn, null, new PagedBytes(PAGED_BYTES_BITS), size * maxDoc, ValueType.BYTES_FIXED_STRAIGHT);
-        this.size = size;
-        this.maxDoc = maxDoc;
-      }
-
-      @Override
-      public BytesRef getBytes(int docID, BytesRef bytesRef) {
-        return data.fillSlice(bytesRef, docID * size, size);
-      }
-      
-      @Override
-      public int getValueCount() {
-        return maxDoc;
-      }
-
-      @Override
-      protected int maxDoc() {
-        return maxDoc;
+    public SingleByteSource(IndexInput datIn, int maxDoc) throws IOException {
+      super(ValueType.BYTES_FIXED_STRAIGHT);
+      try {
+        data = new byte[maxDoc];
+        datIn.readBytes(data, 0, data.length, false);
+      } finally {
+        IOUtils.close(datIn);
       }
     }
-
+    
     @Override
-    public ValuesEnum getEnum(AttributeSource source) throws IOException {
-      return new FixedStraightBytesEnum(source, cloneData(), size, maxDoc);
+    public boolean hasArray() {
+      return true;
     }
 
-   
+    @Override
+    public Object getArray() {
+      return data;
+    }
 
     @Override
-    public ValueType type() {
-      return ValueType.BYTES_FIXED_STRAIGHT;
+    public BytesRef getBytes(int docID, BytesRef bytesRef) {
+      bytesRef.length = 1;
+      bytesRef.bytes = data;
+      bytesRef.offset = docID;
+      return bytesRef;
     }
   }
+
   
-  static class FixedStraightBytesEnum extends ValuesEnum {
-    private final IndexInput datIn;
+  private final static class FixedStraightSource extends BytesSourceBase {
     private final int size;
-    private final int maxDoc;
-    private int pos = -1;
-    private final long fp;
-
-    public FixedStraightBytesEnum(AttributeSource source, IndexInput datIn,
-        int size, int maxDoc) throws IOException {
-      super(source, ValueType.BYTES_FIXED_STRAIGHT);
-      this.datIn = datIn;
-      this.size = size;
-      this.maxDoc = maxDoc;
-      bytesRef.grow(size);
-      bytesRef.length = size;
-      bytesRef.offset = 0;
-      fp = datIn.getFilePointer();
-    }
-
-    protected void copyFrom(ValuesEnum valuesEnum) {
-      super.copyFrom(valuesEnum);
-      if (bytesRef.bytes.length < size) {
-        bytesRef.grow(size);
-      }
-      bytesRef.length = size;
-      bytesRef.offset = 0;
-    }
 
-    public void close() throws IOException {
-      datIn.close();
+    public FixedStraightSource(IndexInput datIn, int size, int maxDoc, ValueType type)
+        throws IOException {
+      super(datIn, null, new PagedBytes(PAGED_BYTES_BITS), size * maxDoc,
+          type);
+      this.size = size;
     }
 
     @Override
-    public int advance(int target) throws IOException {
-      if (target >= maxDoc || size == 0) {
-        return pos = NO_MORE_DOCS;
-      }
-      if ((target - 1) != pos) // pos inc == 1
-        datIn.seek(fp + target * size);
-      datIn.readBytes(bytesRef.bytes, 0, size);
-      return pos = target;
+    public BytesRef getBytes(int docID, BytesRef bytesRef) {
+      return data.fillSlice(bytesRef, docID * size, size);
     }
+  }
+  
+  public final static class DirectFixedStraightSource extends DirectSource {
+    private final int size;
 
-    @Override
-    public int docID() {
-      return pos;
+    DirectFixedStraightSource(IndexInput input, int size, ValueType type) {
+      super(input, type);
+      this.size = size;
     }
 
     @Override
-    public int nextDoc() throws IOException {
-      if (pos >= maxDoc) {
-        return pos = NO_MORE_DOCS;
-      }
-      return advance(pos + 1);
+    protected int position(int docID) throws IOException {
+      data.seek(baseOffset + size * docID);
+      return size;
     }
+
   }
 }

Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/values/Floats.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/values/Floats.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/values/Floats.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/values/Floats.java Mon Oct 10 19:58:24 2011
@@ -22,9 +22,9 @@ import org.apache.lucene.index.values.In
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.IOContext;
 import org.apache.lucene.store.IndexInput;
-import org.apache.lucene.util.AttributeSource;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.Counter;
+import org.apache.lucene.util.IOUtils;
 
 /**
  * Exposes {@link Writer} and reader ({@link Source}) for 32 bit and 64 bit
@@ -37,37 +37,47 @@ import org.apache.lucene.util.Counter;
  */
 public class Floats {
   
-  public static Writer getWriter(Directory dir, String id, int precisionBytes,
-      Counter bytesUsed, IOContext context) throws IOException {
-    if (precisionBytes != 4 && precisionBytes != 8) {
-      throw new IllegalArgumentException("precisionBytes must be 4 or 8; got "
-          + precisionBytes);
-    }
-    return new FloatsWriter(dir, id, bytesUsed, context, precisionBytes);
-
+  protected static final String CODEC_NAME = "Floats";
+  protected static final int VERSION_START = 0;
+  protected static final int VERSION_CURRENT = VERSION_START;
+  
+  public static Writer getWriter(Directory dir, String id, Counter bytesUsed,
+      IOContext context, ValueType type) throws IOException {
+    return new FloatsWriter(dir, id, bytesUsed, context, type);
   }
 
-  public static IndexDocValues getValues(Directory dir, String id, int maxDoc, IOContext context)
+  public static IndexDocValues getValues(Directory dir, String id, int maxDoc, IOContext context, ValueType type)
       throws IOException {
-    return new FloatsReader(dir, id, maxDoc, context);
+    return new FloatsReader(dir, id, maxDoc, context, type);
+  }
+  
+  private static int typeToSize(ValueType type) {
+    switch (type) {
+    case FLOAT_32:
+      return 4;
+    case FLOAT_64:
+      return 8;
+    default:
+      throw new IllegalStateException("illegal type " + type);
+    }
   }
   
   final static class FloatsWriter extends FixedStraightBytesImpl.Writer {
+   
     private final int size; 
+    private final IndexDocValuesArray template;
     public FloatsWriter(Directory dir, String id, Counter bytesUsed,
-        IOContext context, int size) throws IOException {
-      super(dir, id, bytesUsed, context);
+        IOContext context, ValueType type) throws IOException {
+      super(dir, id, CODEC_NAME, VERSION_CURRENT, bytesUsed, context);
+      size = typeToSize(type);
       this.bytesRef = new BytesRef(size);
-      this.size = size;
       bytesRef.length = size;
+      template = IndexDocValuesArray.TEMPLATES.get(type);
+      assert template != null;
     }
     
     public void add(int docID, double v) throws IOException {
-      if (size == 8) {
-        bytesRef.copy(Double.doubleToRawLongBits(v));        
-      } else {
-        bytesRef.copy(Float.floatToRawIntBits((float)v));
-      }
+      template.toBytes(v, bytesRef);
       add(docID, bytesRef);
     }
     
@@ -75,20 +85,27 @@ public class Floats {
     public void add(int docID, PerDocFieldValues docValues) throws IOException {
       add(docID, docValues.getFloat());
     }
+    
+    @Override
+    protected boolean tryBulkMerge(IndexDocValues docValues) {
+      // only bulk merge if value type is the same otherwise size differs
+      return super.tryBulkMerge(docValues) && docValues.type() == template.type();
+    }
+    
+    @Override
+    protected void setMergeBytes(int sourceDoc) {
+      final double value = currentMergeSource.getFloat(sourceDoc);
+      template.toBytes(value, bytesRef);
+    }
   }
-
   
-  final static class FloatsReader extends FixedStraightBytesImpl.Reader {
+  final static class FloatsReader extends FixedStraightBytesImpl.FixedStraightReader {
     final IndexDocValuesArray arrayTemplate;
-    FloatsReader(Directory dir, String id, int maxDoc, IOContext context)
+    FloatsReader(Directory dir, String id, int maxDoc, IOContext context, ValueType type)
         throws IOException {
-      super(dir, id, maxDoc, context);
+      super(dir, id, CODEC_NAME, VERSION_CURRENT, maxDoc, context, type);
+      arrayTemplate = IndexDocValuesArray.TEMPLATES.get(type);
       assert size == 4 || size == 8;
-      if (size == 4) {
-        arrayTemplate = new IndexDocValuesArray.FloatValues();
-      } else {
-        arrayTemplate = new IndexDocValuesArray.DoubleValues();
-      }
     }
     
     @Override
@@ -97,19 +114,10 @@ public class Floats {
       try {
         return arrayTemplate.newFromInput(indexInput, maxDoc);
       } finally {
-        indexInput.close();
+        IOUtils.close(indexInput);
       }
     }
     
-    public ValuesEnum getEnum(AttributeSource source) throws IOException {
-      IndexInput indexInput = (IndexInput) datIn.clone();
-      return arrayTemplate.getDirectEnum(source, indexInput, maxDoc);
-    }
-
-    @Override
-    public ValueType type() {
-      return arrayTemplate.type();
-    }
   }
 
 }
\ No newline at end of file

Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/values/IndexDocValues.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/values/IndexDocValues.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/values/IndexDocValues.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/values/IndexDocValues.java Mon Oct 10 19:58:24 2011
@@ -26,7 +26,6 @@ import org.apache.lucene.index.FieldsEnu
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.codecs.Codec;
 import org.apache.lucene.index.codecs.CodecProvider;
-import org.apache.lucene.util.AttributeSource;
 import org.apache.lucene.util.BytesRef;
 
 /**
@@ -34,8 +33,8 @@ import org.apache.lucene.util.BytesRef;
  * value access based on the lucene internal document id. {@link IndexDocValues}
  * exposes two distinct APIs:
  * <ul>
- * <li>via {@link Source} an entirely RAM resident API for random access</li>
- * <li>via {@link ValuesEnum} a disk resident API for sequential access</li>
+ * <li>via {@link #getSource()} providing RAM resident random access</li>
+ * <li>via {@link #getDirectSource()} providing on disk random access</li>
  * </ul> {@link IndexDocValues} are exposed via
  * {@link IndexReader#perDocValues()} on a per-segment basis. For best
  * performance {@link IndexDocValues} should be consumed per-segment just like
@@ -52,47 +51,18 @@ import org.apache.lucene.util.BytesRef;
  * @lucene.experimental
  */
 public abstract class IndexDocValues implements Closeable {
-  /*
-   * TODO: it might be useful to add another Random Access enum for some
-   * implementations like packed ints and only return such a random access enum
-   * if the impl supports random access. For super large segments it might be
-   * useful or even required in certain environements to have disc based random
-   * access
-   */
-  public static final IndexDocValues[] EMPTY_ARRAY = new IndexDocValues[0];
-
-  private SourceCache cache = new SourceCache.DirectSourceCache();
-
-  /**
-   * Returns an iterator that steps through all documents values for this
-   * {@link IndexDocValues} field instance. {@link ValuesEnum} will skip document
-   * without a value if applicable.
-   */
-  public ValuesEnum getEnum() throws IOException {
-    return getEnum(null);
-  }
 
-  /**
-   * Returns an iterator that steps through all documents values for this
-   * {@link IndexDocValues} field instance. {@link ValuesEnum} will skip document
-   * without a value if applicable.
-   * <p>
-   * If an {@link AttributeSource} is supplied to this method the
-   * {@link ValuesEnum} will use the given source to access implementation
-   * related attributes.
-   */
-  public abstract ValuesEnum getEnum(AttributeSource attrSource)
-      throws IOException;
+  public static final IndexDocValues[] EMPTY_ARRAY = new IndexDocValues[0];
 
+  private volatile SourceCache cache = new SourceCache.DirectSourceCache();
+  private final Object cacheLock = new Object();
+  
   /**
    * Loads a new {@link Source} instance for this {@link IndexDocValues} field
    * instance. Source instances returned from this method are not cached. It is
    * the callers responsibility to maintain the instance and release its
    * resources once the source is not needed anymore.
    * <p>
-   * This method will return null iff this {@link IndexDocValues} represent a
-   * {@link SortedSource}.
-   * <p>
    * For managed {@link Source} instances see {@link #getSource()}.
    * 
    * @see #getSource()
@@ -111,63 +81,18 @@ public abstract class IndexDocValues imp
    * from the cache once this {@link IndexDocValues} instance is closed by the
    * {@link IndexReader}, {@link Fields} or {@link FieldsEnum} the
    * {@link IndexDocValues} was created from.
-   * <p>
-   * This method will return null iff this {@link IndexDocValues} represent a
-   * {@link SortedSource}.
    */
   public Source getSource() throws IOException {
     return cache.load(this);
   }
 
   /**
-   * Returns a {@link SortedSource} instance for this {@link IndexDocValues} field
-   * instance like {@link #getSource()}.
-   * <p>
-   * This method will return null iff this {@link IndexDocValues} represent a
-   * {@link Source} instead of a {@link SortedSource}.
-   */
-  public SortedSource getSortedSorted(Comparator<BytesRef> comparator)
-      throws IOException {
-    return cache.loadSorted(this, comparator);
-  }
-  
-  /**
-   * Returns a {@link SortedSource} instance using a default {@link BytesRef}
-   * comparator for this {@link IndexDocValues} field instance like
-   * {@link #getSource()}.
-   * <p>
-   * This method will return null iff this {@link IndexDocValues} represent a
-   * {@link Source} instead of a {@link SortedSource}.
+   * Returns a disk resident {@link Source} instance. Direct Sources are not
+   * cached in the {@link SourceCache} and should not be shared between threads.
    */
-  public SortedSource getSortedSorted() throws IOException {
-    return getSortedSorted(null);
-  }
+  public abstract Source getDirectSource() throws IOException;
 
   /**
-   * Loads and returns a {@link SortedSource} instance for this
-   * {@link IndexDocValues} field instance like {@link #load()}.
-   * <p>
-   * This method will return null iff this {@link IndexDocValues} represent a
-   * {@link Source} instead of a {@link SortedSource}.
-   */
-  public SortedSource loadSorted(Comparator<BytesRef> comparator)
-      throws IOException {
-    throw new UnsupportedOperationException();
-  }
-  
-  /**
-   * Loads and returns a {@link SortedSource} instance using a default
-   * {@link BytesRef} comparator for this {@link IndexDocValues} field instance
-   * like {@link #load()}.
-   * <p>
-   * This method will return null iff this {@link IndexDocValues} represent a
-   * {@link Source} instead of a {@link SortedSource}.
-   */
-  public SortedSource loadSorted() throws IOException {
-    return loadSorted(null);
-  }
-  
-  /**
    * Returns the {@link ValueType} of this {@link IndexDocValues} instance
    */
   public abstract ValueType type();
@@ -182,14 +107,22 @@ public abstract class IndexDocValues imp
   }
 
   /**
+   * Returns the size per value in bytes or <code>-1</code> iff size per value
+   * is variable.
+   * 
+   * @return the size per value in bytes or <code>-1</code> iff size per value
+   * is variable.
+   */
+  public int getValueSize() {
+    return -1;
+  }
+
+  /**
    * Sets the {@link SourceCache} used by this {@link IndexDocValues} instance. This
-   * method should be called before {@link #load()} or
-   * {@link #loadSorted(Comparator)} is called. All {@link Source} or
-   * {@link SortedSource} instances in the currently used cache will be closed
+   * method should be called before {@link #load()} is called. All {@link Source} instances in the currently used cache will be closed
    * before the new cache is installed.
    * <p>
-   * Note: All instances previously obtained from {@link #load()} or
-   * {@link #loadSorted(Comparator)} will be closed.
+   * Note: All instances previously obtained from {@link #load()} will be lost.
    * 
    * @throws IllegalArgumentException
    *           if the given cache is <code>null</code>
@@ -198,9 +131,10 @@ public abstract class IndexDocValues imp
   public void setCache(SourceCache cache) {
     if (cache == null)
       throw new IllegalArgumentException("cache must not be null");
-    synchronized (this.cache) {
-      this.cache.close(this);
+    synchronized (cacheLock) {
+      SourceCache toClose = this.cache;
       this.cache = cache;
+      toClose.close(this);
     }
   }
 
@@ -208,12 +142,17 @@ public abstract class IndexDocValues imp
    * Source of per document values like long, double or {@link BytesRef}
    * depending on the {@link IndexDocValues} fields {@link ValueType}. Source
    * implementations provide random access semantics similar to array lookups
-   * and typically are entirely memory resident.
    * <p>
-   * {@link Source} defines 3 {@link ValueType} //TODO finish this
+   * @see IndexDocValues#getSource()
+   * @see IndexDocValues#getDirectSource()
    */
   public static abstract class Source {
+    
+    protected final ValueType type;
 
+    protected Source(ValueType type) {
+      this.type = type;
+    }
     /**
      * Returns a <tt>long</tt> for the given document id or throws an
      * {@link UnsupportedOperationException} if this source doesn't support
@@ -242,6 +181,7 @@ public abstract class IndexDocValues imp
      * Returns a {@link BytesRef} for the given document id or throws an
      * {@link UnsupportedOperationException} if this source doesn't support
      * <tt>byte[]</tt> values.
+     * @throws IOException 
      * 
      * @throws UnsupportedOperationException
      *           if this source doesn't support <tt>byte[]</tt> values.
@@ -251,35 +191,15 @@ public abstract class IndexDocValues imp
     }
 
     /**
-     * Returns number of unique values. Some implementations may throw
-     * UnsupportedOperationException.
-     */
-    public int getValueCount() {
-      throw new UnsupportedOperationException();
-    }
-
-    /**
-     * Returns a {@link ValuesEnum} for this source.
-     */
-    public ValuesEnum getEnum() throws IOException {
-      return getEnum(null);
-    }
-
-    /**
      * Returns the {@link ValueType} of this source.
      * 
      * @return the {@link ValueType} of this source.
      */
-    public abstract ValueType type();
+    public ValueType type() {
+      return type;
+    }
 
     /**
-     * Returns a {@link ValuesEnum} for this source which uses the given
-     * {@link AttributeSource}.
-     */
-    public abstract ValuesEnum getEnum(AttributeSource attrSource)
-        throws IOException;
-    
-    /**
      * Returns <code>true</code> iff this {@link Source} exposes an array via
      * {@link #getArray()} otherwise <code>false</code>.
      * 
@@ -297,61 +217,29 @@ public abstract class IndexDocValues imp
     public Object getArray() {
       return null;
     }
-  }
-
-  /**
-   * {@link ValuesEnum} utility for {@link Source} implemenations.
-   * 
-   */
-  public abstract static class SourceEnum extends ValuesEnum {
-    protected final Source source;
-    protected final int numDocs;
-    protected int pos = -1;
-
+    
     /**
-     * Creates a new {@link SourceEnum}
-     * 
-     * @param attrs
-     *          the {@link AttributeSource} for this enum
-     * @param type
-     *          the enums {@link ValueType}
-     * @param source
-     *          the source this enum operates on
-     * @param numDocs
-     *          the number of documents within the source
+     * If this {@link Source} is sorted this method will return an instance of
+     * {@link SortedSource} otherwise <code>null</code>
      */
-    protected SourceEnum(AttributeSource attrs, ValueType type, Source source,
-        int numDocs) {
-      super(attrs, type);
-      this.source = source;
-      this.numDocs = numDocs;
-    }
-
-    @Override
-    public void close() throws IOException {
-    }
-
-    @Override
-    public int docID() {
-      return pos;
-    }
-
-    @Override
-    public int nextDoc() throws IOException {
-      if (pos == NO_MORE_DOCS)
-        return NO_MORE_DOCS;
-      return advance(pos + 1);
+    public SortedSource asSortedSource() {
+      return null;
     }
   }
-
+  
   /**
    * A sorted variant of {@link Source} for <tt>byte[]</tt> values per document.
    * <p>
-   * Note: {@link ValuesEnum} obtained from a {@link SortedSource} will
-   * enumerate values in document order and not in sorted order.
    */
   public static abstract class SortedSource extends Source {
 
+    private final Comparator<BytesRef> comparator;
+
+    protected SortedSource(ValueType type, Comparator<BytesRef> comparator) {
+      super(type);
+      this.comparator = comparator;
+    }
+
     @Override
     public BytesRef getBytes(int docID, BytesRef bytesRef) {
       final int ord = ord(docID);
@@ -364,8 +252,7 @@ public abstract class IndexDocValues imp
     }
 
     /**
-     * Returns ord for specified docID. If this docID had not been added to the
-     * Writer, the ord is 0. Ord is dense, ie, starts at 0, then increments by 1
+     * Returns ord for specified docID. Ord is dense, ie, starts at 0, then increments by 1
      * for the next (as defined by {@link Comparator} value.
      */
     public abstract int ord(int docID);
@@ -373,28 +260,13 @@ public abstract class IndexDocValues imp
     /** Returns value for specified ord. */
     public abstract BytesRef getByOrd(int ord, BytesRef bytesRef);
 
-
-    /**
-     * Finds the ordinal whose value is greater or equal to the given value.
-     * 
-     * @return the given values ordinal if found or otherwise
-     *         <code>(-(ord)-1)</code>, defined as the ordinal of the first
-     *         element that is greater than the given value. This guarantees
-     *         that the return value will always be &gt;= 0 if the given value
-     *         is found.
-     * 
-     */
-    public final int getByValue(BytesRef value) {
-      return getByValue(value, new BytesRef());
-    }
-
     /**
      * Performs a lookup by value.
      * 
      * @param value
      *          the value to look up
-     * @param tmpRef
-     *          a temporary {@link BytesRef} instance used to compare internal
+     * @param spare
+     *          a spare {@link BytesRef} instance used to compare internal
      *          values to the given value. Must not be <code>null</code>
      * @return the given values ordinal if found or otherwise
      *         <code>(-(ord)-1)</code>, defined as the ordinal of the first
@@ -402,6 +274,37 @@ public abstract class IndexDocValues imp
      *         that the return value will always be &gt;= 0 if the given value
      *         is found.
      */
-    public abstract int getByValue(BytesRef value, BytesRef tmpRef);
+    public int getByValue(BytesRef value, BytesRef spare) {
+      return binarySearch(value, spare, 0, getValueCount() - 1);
+    }    
+
+    protected int binarySearch(BytesRef b, BytesRef bytesRef, int low,
+        int high) {
+      int mid = 0;
+      while (low <= high) {
+        mid = (low + high) >>> 1;
+        getByOrd(mid, bytesRef);
+        final int cmp = comparator.compare(bytesRef, b);
+        if (cmp < 0) {
+          low = mid + 1;
+        } else if (cmp > 0) {
+          high = mid - 1;
+        } else {
+          return mid;
+        }
+      }
+      assert comparator.compare(bytesRef, b) != 0;
+      return -(low + 1);
+    }
+    
+    @Override
+    public SortedSource asSortedSource() {
+      return this;
+    }
+    
+    /**
+     * Returns the number of unique values in this sorted source
+     */
+    public abstract int getValueCount();
   }
 }

Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/values/IndexDocValuesArray.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/values/IndexDocValuesArray.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/values/IndexDocValuesArray.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/values/IndexDocValuesArray.java Mon Oct 10 19:58:24 2011
@@ -1,12 +1,12 @@
 package org.apache.lucene.index.values;
 
 import java.io.IOException;
+import java.util.Collections;
+import java.util.EnumMap;
+import java.util.Map;
 
-import org.apache.lucene.index.values.FixedStraightBytesImpl.FixedStraightBytesEnum;
 import org.apache.lucene.index.values.IndexDocValues.Source;
-import org.apache.lucene.index.values.IndexDocValues.SourceEnum;
 import org.apache.lucene.store.IndexInput;
-import org.apache.lucene.util.AttributeSource;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.RamUsageEstimator;
 
@@ -32,82 +32,41 @@ import org.apache.lucene.util.RamUsageEs
  */
 abstract class IndexDocValuesArray extends Source {
 
+  static final Map<ValueType, IndexDocValuesArray> TEMPLATES;
+
+  static {
+    EnumMap<ValueType, IndexDocValuesArray> templates = new EnumMap<ValueType, IndexDocValuesArray>(
+        ValueType.class);
+    templates.put(ValueType.FIXED_INTS_16, new ShortValues());
+    templates.put(ValueType.FIXED_INTS_32, new IntValues());
+    templates.put(ValueType.FIXED_INTS_64, new LongValues());
+    templates.put(ValueType.FIXED_INTS_8, new ByteValues());
+    templates.put(ValueType.FLOAT_32, new FloatValues());
+    templates.put(ValueType.FLOAT_64, new DoubleValues());
+    TEMPLATES = Collections.unmodifiableMap(templates);
+  }
+
   protected final int bytesPerValue;
-  private final ValueType type;
-  private final boolean isFloat;
-  protected int maxDocID = -1;
 
   IndexDocValuesArray(int bytesPerValue, ValueType type) {
+    super(type);
     this.bytesPerValue = bytesPerValue;
-    this.type = type;
-    switch (type) {
-    case FIXED_INTS_16:
-    case FIXED_INTS_32:
-    case FIXED_INTS_64:
-    case FIXED_INTS_8:
-      isFloat = false;
-      break;
-    case FLOAT_32:
-    case FLOAT_64:
-      isFloat = true;
-      break;
-    default:
-      throw new IllegalStateException("illegal type: " + type);
-
-    }
   }
 
   public abstract IndexDocValuesArray newFromInput(IndexInput input, int numDocs)
       throws IOException;
 
   @Override
-  public final int getValueCount() {
-    return maxDocID + 1;
-  }
-
-  @Override
-  public final ValueType type() {
-    return type;
+  public final boolean hasArray() {
+    return true;
   }
 
-  @Override
-  public ValuesEnum getEnum(AttributeSource attrSource) throws IOException {
-    if (isFloat) {
-      return new SourceEnum(attrSource, type(), this, maxDocID + 1) {
-
-        @Override
-        public int advance(int target) throws IOException {
-          if (target >= numDocs) {
-            return pos = NO_MORE_DOCS;
-          }
-          floatsRef.floats[intsRef.offset] = IndexDocValuesArray.this
-              .getFloat(target);
-          return pos = target;
-        }
-      };
-    } else {
-      return new SourceEnum(attrSource, type(), this, maxDocID + 1) {
-
-        @Override
-        public int advance(int target) throws IOException {
-          if (target >= numDocs) {
-            return pos = NO_MORE_DOCS;
-          }
-          intsRef.ints[intsRef.offset] = IndexDocValuesArray.this
-              .getInt(target);
-          return pos = target;
-        }
-
-      };
-    }
+  void toBytes(long value, BytesRef bytesRef) {
+    bytesRef.copy(value);
   }
 
-  abstract ValuesEnum getDirectEnum(AttributeSource attrSource,
-      IndexInput input, int maxDoc) throws IOException;
-
-  @Override
-  public final boolean hasArray() {
-    return true;
+  void toBytes(double value, BytesRef bytesRef) {
+    bytesRef.copy(Double.doubleToRawLongBits(value));
   }
 
   final static class ByteValues extends IndexDocValuesArray {
@@ -122,7 +81,6 @@ abstract class IndexDocValuesArray exten
       super(1, ValueType.FIXED_INTS_8);
       values = new byte[numDocs];
       input.readBytes(values, 0, values.length, false);
-      maxDocID = numDocs - 1;
     }
 
     @Override
@@ -137,24 +95,15 @@ abstract class IndexDocValuesArray exten
     }
 
     @Override
-    ValuesEnum getDirectEnum(AttributeSource attrSource, IndexInput input,
-        int maxDoc) throws IOException {
-      return new FixedIntsEnum(attrSource, input, type(),
-          bytesPerValue, maxDoc) {
-
-        @Override
-        protected final long toLong(BytesRef bytesRef) {
-          return bytesRef.bytes[bytesRef.offset];
-        }
-      };
-    }
-
-    @Override
     public IndexDocValuesArray newFromInput(IndexInput input, int numDocs)
         throws IOException {
       return new ByteValues(input, numDocs);
     }
 
+    void toBytes(long value, BytesRef bytesRef) {
+      bytesRef.bytes[0] = (byte) (0xFFL & value);
+    }
+
   };
 
   final static class ShortValues extends IndexDocValuesArray {
@@ -171,7 +120,6 @@ abstract class IndexDocValuesArray exten
       for (int i = 0; i < values.length; i++) {
         values[i] = input.readShort();
       }
-      maxDocID = numDocs - 1;
     }
 
     @Override
@@ -186,24 +134,15 @@ abstract class IndexDocValuesArray exten
     }
 
     @Override
-    ValuesEnum getDirectEnum(AttributeSource attrSource, IndexInput input,
-        int maxDoc) throws IOException {
-      return new FixedIntsEnum(attrSource, input, type(),
-          bytesPerValue, maxDoc) {
-
-        @Override
-        protected final long toLong(BytesRef bytesRef) {
-          return bytesRef.asShort();
-        }
-      };
-    }
-
-    @Override
     public IndexDocValuesArray newFromInput(IndexInput input, int numDocs)
         throws IOException {
       return new ShortValues(input, numDocs);
     }
 
+    void toBytes(long value, BytesRef bytesRef) {
+      bytesRef.copy((short) (0xFFFFL & value));
+    }
+
   };
 
   final static class IntValues extends IndexDocValuesArray {
@@ -220,7 +159,6 @@ abstract class IndexDocValuesArray exten
       for (int i = 0; i < values.length; i++) {
         values[i] = input.readInt();
       }
-      maxDocID = numDocs - 1;
     }
 
     @Override
@@ -235,23 +173,15 @@ abstract class IndexDocValuesArray exten
     }
 
     @Override
-    ValuesEnum getDirectEnum(AttributeSource attrSource, IndexInput input,
-        int maxDoc) throws IOException {
-      return new FixedIntsEnum(attrSource, input, type(),
-          bytesPerValue, maxDoc) {
-        @Override
-        protected final long toLong(BytesRef bytesRef) {
-          return bytesRef.asInt();
-        }
-      };
-    }
-
-    @Override
     public IndexDocValuesArray newFromInput(IndexInput input, int numDocs)
         throws IOException {
       return new IntValues(input, numDocs);
     }
 
+    void toBytes(long value, BytesRef bytesRef) {
+      bytesRef.copy((int) (0xFFFFFFFF & value));
+    }
+
   };
 
   final static class LongValues extends IndexDocValuesArray {
@@ -268,7 +198,6 @@ abstract class IndexDocValuesArray exten
       for (int i = 0; i < values.length; i++) {
         values[i] = input.readLong();
       }
-      maxDocID = numDocs - 1;
     }
 
     @Override
@@ -283,18 +212,6 @@ abstract class IndexDocValuesArray exten
     }
 
     @Override
-    ValuesEnum getDirectEnum(AttributeSource attrSource, IndexInput input,
-        int maxDoc) throws IOException {
-      return new FixedIntsEnum(attrSource, input, type(),
-          bytesPerValue, maxDoc) {
-        @Override
-        protected final long toLong(BytesRef bytesRef) {
-          return bytesRef.asLong();
-        }
-      };
-    }
-
-    @Override
     public IndexDocValuesArray newFromInput(IndexInput input, int numDocs)
         throws IOException {
       return new LongValues(input, numDocs);
@@ -313,13 +230,13 @@ abstract class IndexDocValuesArray exten
     private FloatValues(IndexInput input, int numDocs) throws IOException {
       super(RamUsageEstimator.NUM_BYTES_FLOAT, ValueType.FLOAT_32);
       values = new float[numDocs];
-      /* we always read BIG_ENDIAN here since the writer serialized plain bytes
-       * we can simply read the ints / longs
-       * back in using readInt / readLong */
+      /*
+       * we always read BIG_ENDIAN here since the writer serialized plain bytes
+       * we can simply read the ints / longs back in using readInt / readLong
+       */
       for (int i = 0; i < values.length; i++) {
         values[i] = Float.intBitsToFloat(input.readInt());
       }
-      maxDocID = numDocs - 1;
     }
 
     @Override
@@ -332,17 +249,11 @@ abstract class IndexDocValuesArray exten
       assert docID >= 0 && docID < values.length;
       return values[docID];
     }
-
+    
     @Override
-    ValuesEnum getDirectEnum(AttributeSource attrSource, IndexInput input,
-        int maxDoc) throws IOException {
-      return new FloatsEnum(attrSource, input, type(),
-          bytesPerValue, maxDoc) {
-            @Override
-            protected double toDouble(BytesRef bytesRef) {
-              return Float.intBitsToFloat(bytesRef.asInt());
-            }
-      };
+    void toBytes(double value, BytesRef bytesRef) {
+      bytesRef.copy(Float.floatToRawIntBits((float)value));
+
     }
 
     @Override
@@ -351,7 +262,7 @@ abstract class IndexDocValuesArray exten
       return new FloatValues(input, numDocs);
     }
   };
-  
+
   final static class DoubleValues extends IndexDocValuesArray {
     private final double[] values;
 
@@ -363,13 +274,13 @@ abstract class IndexDocValuesArray exten
     private DoubleValues(IndexInput input, int numDocs) throws IOException {
       super(RamUsageEstimator.NUM_BYTES_DOUBLE, ValueType.FLOAT_64);
       values = new double[numDocs];
-      /* we always read BIG_ENDIAN here since the writer serialized plain bytes
-       * we can simply read the ints / longs
-       * back in using readInt / readLong */
+      /*
+       * we always read BIG_ENDIAN here since the writer serialized plain bytes
+       * we can simply read the ints / longs back in using readInt / readLong
+       */
       for (int i = 0; i < values.length; i++) {
         values[i] = Double.longBitsToDouble(input.readLong());
       }
-      maxDocID = numDocs - 1;
     }
 
     @Override
@@ -384,78 +295,11 @@ abstract class IndexDocValuesArray exten
     }
 
     @Override
-    ValuesEnum getDirectEnum(AttributeSource attrSource, IndexInput input,
-        int maxDoc) throws IOException {
-      return new FloatsEnum(attrSource, input, type(),
-          bytesPerValue, maxDoc) {
-            @Override
-            protected double toDouble(BytesRef bytesRef) {
-              return Double.longBitsToDouble(bytesRef.asLong());
-            }
-      };
-    }
-
-    @Override
     public IndexDocValuesArray newFromInput(IndexInput input, int numDocs)
         throws IOException {
       return new DoubleValues(input, numDocs);
     }
-  };
-
-  private abstract static class FixedIntsEnum extends
-      FixedStraightBytesEnum {
-    private final ValueType type;
-
-    private FixedIntsEnum(AttributeSource source, IndexInput dataIn,
-        ValueType type, int bytesPerValue, int maxDoc) throws IOException {
-      super(source, dataIn, bytesPerValue, maxDoc);
-      this.type = type;
-
-    }
 
-    @Override
-    public int advance(int target) throws IOException {
-      final int advance = super.advance(target);
-      if (advance != NO_MORE_DOCS) {
-        intsRef.ints[0] = toLong(this.bytesRef);
-      }
-      return advance;
-    }
-    
-    protected abstract long toLong(BytesRef bytesRef);
-
-    @Override
-    public ValueType type() {
-      return type;
-    }
-
-  }
-  
-  private abstract static class FloatsEnum extends FixedStraightBytesEnum {
-
-    private final ValueType type;
-    FloatsEnum(AttributeSource source, IndexInput dataIn, ValueType type, int bytePerValue, int maxDoc)
-        throws IOException {
-      super(source, dataIn, bytePerValue, maxDoc);
-      this.type = type;
-    }
-    
-    @Override
-    public int advance(int target) throws IOException {
-      final int retVal = super.advance(target);
-      if (retVal != NO_MORE_DOCS) {
-        floatsRef.floats[floatsRef.offset] = toDouble(bytesRef);
-      }
-      return retVal;
-    }
-    
-    protected abstract double toDouble(BytesRef bytesRef);
-
-    @Override
-    public ValueType type() {
-      return type;
-    }
-
-  }
+  };
 
 }

Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/values/Ints.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/values/Ints.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/values/Ints.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/values/Ints.java Mon Oct 10 19:58:24 2011
@@ -19,14 +19,9 @@ package org.apache.lucene.index.values;
 
 import java.io.IOException;
 
-import org.apache.lucene.index.values.IndexDocValuesArray.ByteValues;
-import org.apache.lucene.index.values.IndexDocValuesArray.IntValues;
-import org.apache.lucene.index.values.IndexDocValuesArray.LongValues;
-import org.apache.lucene.index.values.IndexDocValuesArray.ShortValues;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.IOContext;
 import org.apache.lucene.store.IndexInput;
-import org.apache.lucene.util.AttributeSource;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.Counter;
 import org.apache.lucene.util.IOUtils;
@@ -37,10 +32,13 @@ import org.apache.lucene.util.IOUtils;
  * @lucene.experimental
  */
 public final class Ints {
+  protected static final String CODEC_NAME = "Ints";
+  protected static final int VERSION_START = 0;
+  protected static final int VERSION_CURRENT = VERSION_START;
 
   private Ints() {
   }
-
+  
   public static Writer getWriter(Directory dir, String id, Counter bytesUsed,
       ValueType type, IOContext context) throws IOException {
     return type == ValueType.VAR_INTS ? new PackedIntValues.PackedIntsWriter(dir, id,
@@ -50,15 +48,42 @@ public final class Ints {
   public static IndexDocValues getValues(Directory dir, String id, int numDocs,
       ValueType type, IOContext context) throws IOException {
     return type == ValueType.VAR_INTS ? new PackedIntValues.PackedIntsReader(dir, id,
-        numDocs, context) : new IntsReader(dir, id, numDocs, context);
+        numDocs, context) : new IntsReader(dir, id, numDocs, context, type);
+  }
+  
+  private static ValueType sizeToType(int size) {
+    switch (size) {
+    case 1:
+      return ValueType.FIXED_INTS_8;
+    case 2:
+      return ValueType.FIXED_INTS_16;
+    case 4:
+      return ValueType.FIXED_INTS_32;
+    case 8:
+      return ValueType.FIXED_INTS_64;
+    default:
+      throw new IllegalStateException("illegal size " + size);
+    }
+  }
+  
+  private static int typeToSize(ValueType type) {
+    switch (type) {
+    case FIXED_INTS_16:
+      return 2;
+    case FIXED_INTS_32:
+      return 4;
+    case FIXED_INTS_64:
+      return 8;
+    case FIXED_INTS_8:
+      return 1;
+    default:
+      throw new IllegalStateException("illegal type " + type);
+    }
   }
 
-  static class IntsWriter extends FixedStraightBytesImpl.Writer {
-    protected static final String CODEC_NAME = "Ints";
-    protected static final int VERSION_START = 0;
-    protected static final int VERSION_CURRENT = VERSION_START;
 
-    private final ValueType valueType;
+  static class IntsWriter extends FixedStraightBytesImpl.Writer {
+    private final IndexDocValuesArray template;
 
     public IntsWriter(Directory dir, String id, Counter bytesUsed,
         IOContext context, ValueType valueType) throws IOException {
@@ -68,46 +93,15 @@ public final class Ints {
     protected IntsWriter(Directory dir, String id, String codecName,
         int version, Counter bytesUsed, IOContext context, ValueType valueType) throws IOException {
       super(dir, id, codecName, version, bytesUsed, context);
-      this.valueType = valueType;
-      final int expectedSize = getSize(valueType);
-      this.bytesRef = new BytesRef(expectedSize);
-      bytesRef.length = expectedSize;
+      size = typeToSize(valueType);
+      this.bytesRef = new BytesRef(size);
+      bytesRef.length = size;
+      template = IndexDocValuesArray.TEMPLATES.get(valueType);
     }
     
-    private static int getSize(ValueType type) {
-      switch (type) {
-      case FIXED_INTS_16:
-        return 2;
-      case FIXED_INTS_32:
-        return 4;
-      case FIXED_INTS_64:
-        return 8;
-      case FIXED_INTS_8:
-        return 1;
-      default:
-        throw new IllegalStateException("illegal type " + type);
-      }
-    }
-
     @Override
     public void add(int docID, long v) throws IOException {
-      switch (valueType) {
-      case FIXED_INTS_64:
-        bytesRef.copy(v);
-        break;
-      case FIXED_INTS_32:
-        bytesRef.copy((int) (0xFFFFFFFF & v));
-        break;
-      case FIXED_INTS_16:
-        bytesRef.copy((short) (0xFFFFL & v));
-        break;
-      case FIXED_INTS_8:
-        bytesRef.bytes[0] = (byte) (0xFFL & v);
-        break;
-      default:
-        throw new IllegalStateException("illegal type " + valueType);
-      }
-
+      template.toBytes(v, bytesRef);
       add(docID, bytesRef);
     }
 
@@ -115,73 +109,40 @@ public final class Ints {
     public void add(int docID, PerDocFieldValues docValues) throws IOException {
       add(docID, docValues.getInt());
     }
+    
+    @Override
+    protected void setMergeBytes(int sourceDoc) {
+      final long value = currentMergeSource.getInt(sourceDoc);
+      template.toBytes(value, bytesRef);
+    }
+    
+    @Override
+    protected boolean tryBulkMerge(IndexDocValues docValues) {
+      // only bulk merge if value type is the same otherwise size differs
+      return super.tryBulkMerge(docValues) && docValues.type() == template.type();
+    }
   }
-
-  final static class IntsReader extends FixedStraightBytesImpl.Reader {
-    private final ValueType type;
+  
+  final static class IntsReader extends FixedStraightBytesImpl.FixedStraightReader {
     private final IndexDocValuesArray arrayTemplate;
 
-    IntsReader(Directory dir, String id, int maxDoc, IOContext context)
+    IntsReader(Directory dir, String id, int maxDoc, IOContext context, ValueType type)
         throws IOException {
-      super(dir, id, IntsWriter.CODEC_NAME, IntsWriter.VERSION_CURRENT, maxDoc,
-          context);
-      switch (size) {
-      case 8:
-        type = ValueType.FIXED_INTS_64;
-        arrayTemplate = new LongValues();
-        break;
-      case 4:
-        type = ValueType.FIXED_INTS_32;
-        arrayTemplate = new IntValues();
-        break;
-      case 2:
-        type = ValueType.FIXED_INTS_16;
-        arrayTemplate = new ShortValues();
-        break;
-      case 1:
-        type = ValueType.FIXED_INTS_8;
-        arrayTemplate = new ByteValues();
-        break;
-      default:
-        throw new IllegalStateException("illegal size: " + size);
-      }
+      super(dir, id, CODEC_NAME, VERSION_CURRENT, maxDoc,
+          context, type);
+      arrayTemplate = IndexDocValuesArray.TEMPLATES.get(type);
+      assert arrayTemplate != null;
+      assert type == sizeToType(size);
     }
 
     @Override
     public Source load() throws IOException {
-      boolean success = false;
-      IndexInput input = null;
-      try {
-        input = cloneData();
-        final Source source = arrayTemplate.newFromInput(input, maxDoc);
-        success = true;
-        return source;
-      } finally {
-        if (!success) {
-          IOUtils.closeWhileHandlingException(input, datIn);
-        }
-      }
-    }
-
-    @Override
-    public ValuesEnum getEnum(AttributeSource source) throws IOException {
-      final IndexInput input = cloneData();
-      boolean success = false;
+      final IndexInput indexInput = cloneData();
       try {
-        final ValuesEnum valuesEnum = arrayTemplate.getDirectEnum(source,
-            input, maxDoc);
-        success = true;
-        return valuesEnum;
+        return arrayTemplate.newFromInput(indexInput, maxDoc);
       } finally {
-        if (!success) {
-          IOUtils.closeWhileHandlingException(input);
-        }
+        IOUtils.close(indexInput);
       }
     }
-
-    @Override
-    public ValueType type() {
-      return type;
-    }
   }
 }



Mime
View raw message