lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mikemcc...@apache.org
Subject svn commit: r1430064 - in /lucene/dev/branches/lucene4547/lucene: codecs/src/java/org/apache/lucene/codecs/memory/ codecs/src/java/org/apache/lucene/codecs/simpletext/ core/src/java/org/apache/lucene/codecs/ core/src/java/org/apache/lucene/codecs/lucen...
Date Mon, 07 Jan 2013 22:29:34 GMT
Author: mikemccand
Date: Mon Jan  7 22:29:33 2013
New Revision: 1430064

URL: http://svn.apache.org/viewvc?rev=1430064&view=rev
Log:
cutover to Iterable<BytesRef> for BinaryDV

Removed:
    lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/BinaryDocValuesConsumer.java
    lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41BinaryDocValues.java
    lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41BinaryDocValuesConsumer.java
Modified:
    lucene/dev/branches/lucene4547/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesFormat.java
    lucene/dev/branches/lucene4547/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSimpleDocValuesFormat.java
    lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/SimpleDVConsumer.java
    lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41SimpleDocValuesFormat.java
    lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41DocValuesConsumer.java
    lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41DocValuesProducer.java
    lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41SortedDocValues.java
    lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldDocValuesFormat.java
    lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/BinaryDocValues.java
    lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/BytesDVWriter.java
    lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
    lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/SortedDocValues.java
    lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/search/FieldCacheImpl.java
    lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/TestDemoDocValue.java
    lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/index/TestDuelingCodecs.java
    lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/search/TestFieldCache.java
    lucene/dev/branches/lucene4547/lucene/test-framework/src/java/org/apache/lucene/index/MultiSimpleDocValues.java

Modified: lucene/dev/branches/lucene4547/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesFormat.java?rev=1430064&r1=1430063&r2=1430064&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesFormat.java
(original)
+++ lucene/dev/branches/lucene4547/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesFormat.java
Mon Jan  7 22:29:33 2013
@@ -94,8 +94,6 @@ public class MemoryDocValuesFormat exten
       public BinaryDocValues getBinary(FieldInfo field) throws IOException {
         BinaryDocValues valuesIn = producer.getBinary(field);
         final int maxDoc = valuesIn.size();
-        final int maxLength = valuesIn.maxLength();
-        final boolean fixedLength = valuesIn.isFixedLength();
         // nocommit more ram efficient
         final byte[][] values = new byte[maxDoc][];
         BytesRef scratch = new BytesRef();
@@ -118,16 +116,6 @@ public class MemoryDocValuesFormat exten
           public int size() {
             return maxDoc;
           }
-
-          @Override
-          public boolean isFixedLength() {
-            return fixedLength;
-          }
-
-          @Override
-          public int maxLength() {
-            return maxLength;
-          }
         };
       }
 

Modified: lucene/dev/branches/lucene4547/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSimpleDocValuesFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSimpleDocValuesFormat.java?rev=1430064&r1=1430063&r2=1430064&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSimpleDocValuesFormat.java
(original)
+++ lucene/dev/branches/lucene4547/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSimpleDocValuesFormat.java
Mon Jan  7 22:29:33 2013
@@ -29,7 +29,6 @@ import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
 
-import org.apache.lucene.codecs.BinaryDocValuesConsumer;
 import org.apache.lucene.codecs.SimpleDVConsumer;
 import org.apache.lucene.codecs.SimpleDVProducer;
 import org.apache.lucene.codecs.SimpleDocValuesFormat;
@@ -63,10 +62,10 @@ public class SimpleTextSimpleDocValuesFo
   final static BytesRef MINVALUE = new BytesRef("  minvalue ");
   final static BytesRef PATTERN  = new BytesRef("  pattern ");
   // used for bytes
-  final static BytesRef FIXEDLENGTH = new BytesRef("  fixedlength ");
-  final static BytesRef MAXLENGTH = new BytesRef("  maxlength ");
   final static BytesRef LENGTH = new BytesRef("length ");
+  final static BytesRef MAXLENGTH = new BytesRef("  maxlength ");
   // used for sorted bytes
+  final static BytesRef FIXEDLENGTH = new BytesRef("  fixedlength ");
   final static BytesRef NUMVALUES = new BytesRef("  numvalues ");
   final static BytesRef ORDPATTERN = new BytesRef("  ordpattern ");
   
@@ -101,8 +100,7 @@ public class SimpleTextSimpleDocValuesFo
    *  for bytes this is also a "fixed-width" file, for example:
    *  <pre>
    *  field myField
-   *    fixedlength false
-   *    maxlength 8
+   *    maxlength 6
    *    pattern 0
    *  length 6
    *  foobar[space][space]
@@ -221,15 +219,16 @@ public class SimpleTextSimpleDocValuesFo
     }
 
     @Override
-    public BinaryDocValuesConsumer addBinaryField(FieldInfo field, boolean fixedLength, final
int maxLength) throws IOException {
+    public void addBinaryField(FieldInfo field, Iterable<BytesRef> values) throws IOException
{
       assert fieldSeen(field.name);
       assert DocValues.isBytes(field.getDocValuesType());
       assert !isNorms;
+      int maxLength = 0;
+      for(BytesRef value : values) {
+        maxLength = Math.max(maxLength, value.length);
+      }
       writeFieldEntry(field);
-      // write fixedlength
-      SimpleTextUtil.write(data, FIXEDLENGTH);
-      SimpleTextUtil.write(data, Boolean.toString(fixedLength), scratch);
-      SimpleTextUtil.writeNewline(data);
+
       // write maxLength
       SimpleTextUtil.write(data, MAXLENGTH);
       SimpleTextUtil.write(data, Integer.toString(maxLength), scratch);
@@ -245,34 +244,27 @@ public class SimpleTextSimpleDocValuesFo
       SimpleTextUtil.write(data, sb.toString(), scratch);
       SimpleTextUtil.writeNewline(data);
       final DecimalFormat encoder = new DecimalFormat(sb.toString(), new DecimalFormatSymbols(Locale.ROOT));
-      
-      return new BinaryDocValuesConsumer() {
-        int numDocsWritten = 0;
-        
-        @Override
-        public void add(BytesRef value) throws IOException {
-          // write length
-          SimpleTextUtil.write(data, LENGTH);
-          SimpleTextUtil.write(data, encoder.format(value.length), scratch);
-          SimpleTextUtil.writeNewline(data);
-          
-          // write bytes -- don't use SimpleText.write
-          // because it escapes:
-          data.writeBytes(value.bytes, value.offset, value.length);
 
-          // pad to fit
-          for (int i = value.length; i < maxLength; i++) {
-            data.writeByte((byte)' ');
-          }
-          SimpleTextUtil.writeNewline(data);
-          numDocsWritten++;
+      int numDocsWritten = 0;
+      for(BytesRef value : values) {
+        // write length
+        SimpleTextUtil.write(data, LENGTH);
+        SimpleTextUtil.write(data, encoder.format(value.length), scratch);
+        SimpleTextUtil.writeNewline(data);
+          
+        // write bytes -- don't use SimpleText.write
+        // because it escapes:
+        data.writeBytes(value.bytes, value.offset, value.length);
+
+        // pad to fit
+        for (int i = value.length; i < maxLength; i++) {
+          data.writeByte((byte)' ');
         }
+        SimpleTextUtil.writeNewline(data);
+        numDocsWritten++;
+      }
 
-        @Override
-        public void finish() throws IOException {
-          assert numDocs == numDocsWritten;
-        }
-      };
+      assert numDocs == numDocsWritten;
     }
     
     @Override
@@ -446,9 +438,6 @@ public class SimpleTextSimpleDocValuesFo
           data.seek(data.getFilePointer() + (1+field.pattern.length()) * maxDoc);
         } else if (DocValues.isBytes(dvType)) {
           readLine();
-          assert startsWith(FIXEDLENGTH);
-          field.fixedLength = Boolean.parseBoolean(stripPrefix(FIXEDLENGTH));
-          readLine();
           assert startsWith(MAXLENGTH);
           field.maxLength = Integer.parseInt(stripPrefix(MAXLENGTH));
           readLine();
@@ -574,16 +563,6 @@ public class SimpleTextSimpleDocValuesFo
         public int size() {
           return maxDoc;
         }
-
-        @Override
-        public boolean isFixedLength() {
-          return field.fixedLength;
-        }
-
-        @Override
-        public int maxLength() {
-          return field.maxLength;
-        }
       };
     }
 

Modified: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/SimpleDVConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/SimpleDVConsumer.java?rev=1430064&r1=1430063&r2=1430064&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/SimpleDVConsumer.java
(original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/SimpleDVConsumer.java
Mon Jan  7 22:29:33 2013
@@ -40,7 +40,8 @@ public abstract class SimpleDVConsumer i
   // nocommit should we pass SegmentWriteState...?
   public abstract void addNumericField(FieldInfo field, Iterable<Number> values) throws
IOException;    
 
-  public abstract BinaryDocValuesConsumer addBinaryField(FieldInfo field, boolean fixedLength,
int maxLength) throws IOException;
+  public abstract void addBinaryField(FieldInfo field, Iterable<BytesRef> values) throws
IOException;
+
   // nocommit: figure out whats fair here.
   public abstract SortedDocValuesConsumer addSortedField(FieldInfo field, int valueCount,
boolean fixedLength, int maxLength) throws IOException;
 
@@ -114,34 +115,72 @@ public abstract class SimpleDVConsumer i
   }
   
   // dead simple impl: codec can optimize
-  public void mergeBinaryField(FieldInfo fieldInfo, MergeState mergeState, List<BinaryDocValues>
toMerge) throws IOException {
-    // first compute fixedLength and maxLength of live ones to be merged.
-    // nocommit: messy, and can be simplified by using docValues.maxLength/fixedLength in
many cases.
-    boolean fixedLength = true;
-    int maxLength = -1;
-    BytesRef bytes = new BytesRef();
-    for (int readerIDX=0;readerIDX<toMerge.size();readerIDX++) {
-      AtomicReader reader = mergeState.readers.get(readerIDX);      
-      int maxDoc = reader.maxDoc();
-      Bits liveDocs = reader.getLiveDocs();
-      BinaryDocValues values = toMerge.get(readerIDX);
-      for (int i = 0; i < maxDoc; i++) {
-        if (liveDocs == null || liveDocs.get(i)) {
-          values.get(i, bytes);
-          if (maxLength == -1) {
-            maxLength = bytes.length;
-          } else {
-            fixedLength &= bytes.length == maxLength;
-            maxLength = Math.max(bytes.length, maxLength);
-          }
-        }
-        mergeState.checkAbort.work(300);
-      }
-    }
-    // now we can merge
-    assert maxLength >= 0; // could this happen (nothing to do?)
-    BinaryDocValuesConsumer field = addBinaryField(fieldInfo, fixedLength, maxLength);
-    field.merge(mergeState, toMerge);
+  public void mergeBinaryField(FieldInfo fieldInfo, final MergeState mergeState, final List<BinaryDocValues>
toMerge) throws IOException {
+
+    addBinaryField(fieldInfo,
+                   new Iterable<BytesRef>() {
+                     @Override
+                     public Iterator<BytesRef> iterator() {
+                       return new Iterator<BytesRef>() {
+                         int readerUpto = -1;
+                         int docIDUpto;
+                         BytesRef nextValue = new BytesRef();
+                         AtomicReader currentReader;
+                         BinaryDocValues currentValues;
+                         Bits currentLiveDocs;
+                         boolean nextIsSet;
+
+                         @Override
+                         public boolean hasNext() {
+                           return nextIsSet || setNext();
+                         }
+
+                         @Override
+                         public void remove() {
+                           throw new UnsupportedOperationException();
+                         }
+
+                         @Override
+                         public BytesRef next() {
+                           if (!hasNext()) {
+                             throw new NoSuchElementException();
+                           }
+                           assert nextIsSet;
+                           nextIsSet = false;
+                           // nocommit make a mutable number
+                           return nextValue;
+                         }
+
+                         private boolean setNext() {
+                           while (true) {
+                             if (readerUpto == toMerge.size()) {
+                               return false;
+                             }
+
+                             if (currentReader == null || docIDUpto == currentReader.maxDoc())
{
+                               readerUpto++;
+                               if (readerUpto < toMerge.size()) {
+                                 currentReader = mergeState.readers.get(readerUpto);
+                                 currentValues = toMerge.get(readerUpto);
+                                 currentLiveDocs = currentReader.getLiveDocs();
+                               }
+                               docIDUpto = 0;
+                               continue;
+                             }
+
+                             if (currentLiveDocs == null || currentLiveDocs.get(docIDUpto))
{
+                               nextIsSet = true;
+                               currentValues.get(docIDUpto, nextValue);
+                               docIDUpto++;
+                               return true;
+                             }
+
+                             docIDUpto++;
+                           }
+                         }
+                       };
+                     }
+                   });
   }
 
   public void mergeSortedField(FieldInfo fieldInfo, MergeState mergeState, List<SortedDocValues>
toMerge) throws IOException {

Modified: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41SimpleDocValuesFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41SimpleDocValuesFormat.java?rev=1430064&r1=1430063&r2=1430064&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41SimpleDocValuesFormat.java
(original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41SimpleDocValuesFormat.java
Mon Jan  7 22:29:33 2013
@@ -21,7 +21,6 @@ import java.io.IOException;
 import java.util.HashMap;
 import java.util.Map;
 
-import org.apache.lucene.codecs.BinaryDocValuesConsumer;
 import org.apache.lucene.codecs.CodecUtil;
 import org.apache.lucene.codecs.SimpleDVConsumer;
 import org.apache.lucene.codecs.SimpleDVProducer;
@@ -108,23 +107,23 @@ public class Lucene41SimpleDocValuesForm
     }
 
     @Override
-    public BinaryDocValuesConsumer addBinaryField(FieldInfo field, boolean fixedLength, int
maxLength) throws IOException {
+    public void addBinaryField(FieldInfo field, Iterable<BytesRef> values) throws IOException
{
       meta.writeVInt(field.number);
-      meta.writeByte(fixedLength ? (byte)1 : 0);
-      meta.writeVInt(maxLength);
+      // nocommit handle var length too!!
+      int length = -1;
+      for(BytesRef v : values) {
+        if (length == -1) {
+          length = v.length;
+        } else if (length != v.length) {
+          throw new UnsupportedOperationException();
+        }
+      }
+      // nocommit don't hardwire fixedLength to 1:
+      meta.writeByte((byte) 1);
+      meta.writeVInt(length);
       meta.writeLong(data.getFilePointer());
-      if (fixedLength) {
-        return new BinaryDocValuesConsumer() {
-          @Override
-          public void add(BytesRef value) throws IOException {
-            data.writeBytes(value.bytes, value.offset, value.length);
-          }
-
-          @Override
-          public void finish() throws IOException {}
-        };
-      } else {
-        throw new UnsupportedOperationException();
+      for(BytesRef value : values) {
+        data.writeBytes(value.bytes, value.offset, value.length);
       }
     }
 
@@ -267,16 +266,6 @@ public class Lucene41SimpleDocValuesForm
         }
 
         @Override
-        public boolean isFixedLength() {
-          return entry.fixedLength;
-        }
-
-        @Override
-        public int maxLength() {
-          return entry.maxLength;
-        }
-        
-        @Override
         public int size() {
           return maxDoc;
         }

Modified: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41DocValuesConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41DocValuesConsumer.java?rev=1430064&r1=1430063&r2=1430064&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41DocValuesConsumer.java
(original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41DocValuesConsumer.java
Mon Jan  7 22:29:33 2013
@@ -18,7 +18,6 @@ package org.apache.lucene.codecs.lucene4
  */
 import java.io.IOException;
 
-import org.apache.lucene.codecs.BinaryDocValuesConsumer;
 import org.apache.lucene.codecs.SimpleDVConsumer;
 import org.apache.lucene.codecs.SortedDocValuesConsumer;
 import org.apache.lucene.index.FieldInfo;
@@ -29,6 +28,7 @@ import org.apache.lucene.store.CompoundF
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.IOContext;
 import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.IOUtils;
 
 public class Lucene41DocValuesConsumer extends SimpleDVConsumer {
@@ -86,25 +86,8 @@ public class Lucene41DocValuesConsumer e
   }
   
   @Override
-  public BinaryDocValuesConsumer addBinaryField(FieldInfo field,
-      boolean fixedLength, int maxLength) throws IOException {
-    String nameData = getDocValuesFileName(info, field, DATA_EXTENSION);
-    String idxOut = getDocValuesFileName(info, field, INDEX_EXTENSION);
-    boolean success = false;
-    IndexOutput dataOut = null;
-    IndexOutput indexOut = null;
-    try {
-      dataOut = getDirectory().createOutput(nameData, context);
-      indexOut = getDirectory().createOutput(idxOut, context);
-      Lucene41BinaryDocValuesConsumer consumer = new Lucene41BinaryDocValuesConsumer(
-          dataOut, indexOut, fixedLength, maxLength);
-      success = true;
-      return consumer;
-    } finally {
-      if (!success) {
-        IOUtils.close(dataOut, indexOut);
-      }
-    }
+  public void addBinaryField(FieldInfo field, Iterable<BytesRef> values) {
+    // ncommit
   }
   
   // nocommit: bogus to put segmentName in here. think about copySegmentAsIs!!!!!!

Modified: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41DocValuesProducer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41DocValuesProducer.java?rev=1430064&r1=1430063&r2=1430064&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41DocValuesProducer.java
(original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41DocValuesProducer.java
Mon Jan  7 22:29:33 2013
@@ -72,11 +72,8 @@ public class Lucene41DocValuesProducer e
   
   @Override
   public BinaryDocValues getBinary(FieldInfo field) throws IOException {
-    if (DocValues.isBytes(field.getDocValuesType()) || DocValues.isSortedBytes(field.getDocValuesType()))
{
-      return new Lucene41BinaryDocValues.Factory(this.cfs, this.info, field, context).getDirect();
-    } else {
-      return null;
-    }
+    // nocommit
+    return null;
   }
   
   @Override

Modified: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41SortedDocValues.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41SortedDocValues.java?rev=1430064&r1=1430063&r2=1430064&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41SortedDocValues.java
(original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41SortedDocValues.java
Mon Jan  7 22:29:33 2013
@@ -146,7 +146,8 @@ public class Lucene41SortedDocValues ext
         this.valueCount = datIn.readInt();
         this.baseOffset = datIn.getFilePointer();
         
-        if (size == Lucene41BinaryDocValuesConsumer.VALUE_SIZE_VAR) {
+        //if (size == Lucene41BinaryDocValuesConsumer.VALUE_SIZE_VAR) {
+        if (size == -1) {
           offsetIn = dir.openInput(Lucene41DocValuesConsumer
               .getDocValuesFileName(segmentInfo, field,
                   Lucene41DocValuesConsumer.OFFSET_EXTENSION), context);

Modified: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldDocValuesFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldDocValuesFormat.java?rev=1430064&r1=1430063&r2=1430064&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldDocValuesFormat.java
(original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldDocValuesFormat.java
Mon Jan  7 22:29:33 2013
@@ -25,7 +25,6 @@ import java.util.Map;
 import java.util.ServiceLoader; // javadocs
 import java.util.TreeMap;
 
-import org.apache.lucene.codecs.BinaryDocValuesConsumer;
 import org.apache.lucene.codecs.PostingsFormat;
 import org.apache.lucene.codecs.SimpleDVConsumer;
 import org.apache.lucene.codecs.SimpleDVProducer;
@@ -37,6 +36,7 @@ import org.apache.lucene.index.NumericDo
 import org.apache.lucene.index.SegmentReadState;
 import org.apache.lucene.index.SegmentWriteState;
 import org.apache.lucene.index.SortedDocValues;
+import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.IOUtils;
 
 /**
@@ -106,8 +106,8 @@ public abstract class PerFieldDocValuesF
     }
 
     @Override
-    public BinaryDocValuesConsumer addBinaryField(FieldInfo field, boolean fixedLength, int
maxLength) throws IOException {
-      return getInstance(field).addBinaryField(field, fixedLength, maxLength);
+    public void addBinaryField(FieldInfo field, Iterable<BytesRef> values) throws IOException
{
+      getInstance(field).addBinaryField(field, values);
     }
 
     @Override

Modified: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/BinaryDocValues.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/BinaryDocValues.java?rev=1430064&r1=1430063&r2=1430064&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/BinaryDocValues.java
(original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/BinaryDocValues.java
Mon Jan  7 22:29:33 2013
@@ -27,9 +27,6 @@ public abstract class BinaryDocValues {
   
   public abstract int size();
   
-  public abstract boolean isFixedLength();
-  public abstract int maxLength();
-  
   public static class EMPTY extends BinaryDocValues {
     private final int size;
     
@@ -46,15 +43,5 @@ public abstract class BinaryDocValues {
     public int size() {
       return size;
     }
-
-    @Override
-    public boolean isFixedLength() {
-      return true;
-    }
-
-    @Override
-    public int maxLength() {
-      return 0;
-    }
   };
 }

Modified: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/BytesDVWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/BytesDVWriter.java?rev=1430064&r1=1430063&r2=1430064&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/BytesDVWriter.java
(original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/BytesDVWriter.java
Mon Jan  7 22:29:33 2013
@@ -18,8 +18,8 @@ package org.apache.lucene.index;
  */
 
 import java.io.IOException;
+import java.util.Iterator;
 
-import org.apache.lucene.codecs.BinaryDocValuesConsumer;
 import org.apache.lucene.codecs.SimpleDVConsumer;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.BytesRefArray;
@@ -37,16 +37,9 @@ class BytesDVWriter extends DocValuesWri
   private int addedValues = 0;
   private final BytesRef emptyBytesRef = new BytesRef();
 
-  // -2 means not set yet; -1 means length isn't fixed;
-  // -otherwise it's the fixed length seen so far:
-  int fixedLength = -2;
-  int maxLength;
-  int totalSize;
-
   public BytesDVWriter(FieldInfo fieldInfo, Counter counter) {
     this.fieldInfo = fieldInfo;
     this.bytesRefArray = new BytesRefArray(counter);
-    this.totalSize = 0;
   }
 
   public void addValue(int docID, BytesRef value) {
@@ -57,54 +50,60 @@ class BytesDVWriter extends DocValuesWri
       // nocommit improve message
       throw new IllegalArgumentException("null binaryValue not allowed (field=" + fieldInfo.name
+ ")");
     }
-    mergeLength(value.length);
     
     // Fill in any holes:
     while(addedValues < docID) {
       addedValues++;
       bytesRefArray.append(emptyBytesRef);
-      mergeLength(0);
     }
     addedValues++;
     bytesRefArray.append(value);
   }
 
-  private void mergeLength(int length) {
-    if (fixedLength == -2) {
-      fixedLength = length;
-    } else if (fixedLength != length) {
-      fixedLength = -1;
-    }
-    maxLength = Math.max(maxLength, length);
-    totalSize += length;
-  }
-
   @Override
   public void finish(int maxDoc) {
-    if (addedValues < maxDoc) {
-      mergeLength(0);
-    }
   }
 
   @Override
   public void flush(SegmentWriteState state, SimpleDVConsumer dvConsumer) throws IOException
{
-    BinaryDocValuesConsumer consumer = dvConsumer.addBinaryField(fieldInfo,
-                                                                 fixedLength >= 0,
-                                                                 maxLength);
-    final int bufferedDocCount = addedValues;
-    BytesRef value = new BytesRef();
-    for(int docID=0;docID<bufferedDocCount;docID++) {
-      bytesRefArray.get(value, docID);
-      consumer.add(value);
-    }
     final int maxDoc = state.segmentInfo.getDocCount();
-    value.length = 0;
-    for(int docID=bufferedDocCount;docID<maxDoc;docID++) {
-      consumer.add(value);
-    }
-    consumer.finish();
+
+    dvConsumer.addBinaryField(fieldInfo,
+                              new Iterable<BytesRef>() {
+
+                                @Override
+                                public Iterator<BytesRef> iterator() {
+                                   return new Iterator<BytesRef>() {
+                                     BytesRef value = new BytesRef();
+                                     int upto;
+
+                                     @Override
+                                     public boolean hasNext() {
+                                       return upto < maxDoc;
+                                     }
+
+                                     @Override
+                                     public void remove() {
+                                       throw new UnsupportedOperationException();
+                                     }
+
+                                     @Override
+                                     public BytesRef next() {
+                                       // nocommit make
+                                       // mutable Number:
+                                       if (upto < bytesRefArray.size()) {
+                                         bytesRefArray.get(value, upto);
+                                       } else {
+                                         value.length = 0;
+                                       }
+                                       upto++;
+                                       return value;
+                                     }
+                                   };
+                                 }
+                               });
+
     reset();
-    //System.out.println("FLUSH");
   }
 
   public void abort() {
@@ -113,7 +112,5 @@ class BytesDVWriter extends DocValuesWri
 
   private void reset() {
     bytesRefArray.clear();
-    fixedLength = -2;
-    maxLength = 0;
   }
 }
\ No newline at end of file

Modified: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java?rev=1430064&r1=1430063&r2=1430064&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
(original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
Mon Jan  7 22:29:33 2013
@@ -1378,25 +1378,10 @@ public class CheckIndex {
     if (dv == null) {
       return;
     }
-    final boolean fixed = dv.isFixedLength();
-    final int maxLength = dv.maxLength();
-    boolean fixed2 = true;
-    int maxLength2 = -1;
+    // nocommit what else to check ...
     BytesRef scratch = new BytesRef();
     for (int i = 0; i < reader.maxDoc(); i++) {
       dv.get(i, scratch);
-      if (maxLength2 == -1) {
-        maxLength2 = scratch.length;
-      } else {
-        fixed2 &= scratch.length == maxLength2;
-        maxLength2 = Math.max(maxLength2, scratch.length);
-      }
-    }
-    if (fixed != fixed2) {
-      throw new RuntimeException("dv for field: " + fieldName + " reports fixed=" + fixed
+ " but this is not the case!");
-    }
-    if (maxLength != maxLength2) {
-      throw new RuntimeException("dv for field: " + fieldName + " reports maxLength=" + maxLength
+ " but this is not the case: " + maxLength2);
     }
   }
   

Modified: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/SortedDocValues.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/SortedDocValues.java?rev=1430064&r1=1430063&r2=1430064&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/SortedDocValues.java
(original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/SortedDocValues.java
Mon Jan  7 22:29:33 2013
@@ -30,6 +30,10 @@ public abstract class SortedDocValues ex
 
   public abstract int getValueCount();
 
+  public abstract boolean isFixedLength();
+
+  public abstract int maxLength();
+
   @Override
   public void get(int docID, BytesRef result) {
     int ord = getOrd(docID);

Modified: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/search/FieldCacheImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/search/FieldCacheImpl.java?rev=1430064&r1=1430063&r2=1430064&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/search/FieldCacheImpl.java
(original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/search/FieldCacheImpl.java
Mon Jan  7 22:29:33 2013
@@ -1290,16 +1290,6 @@ class FieldCacheImpl implements FieldCac
         bytes.fill(ret, pointer);
       }
     }
-
-    @Override
-    public boolean isFixedLength() {
-      return isFixedLength;
-    }
-
-    @Override
-    public int maxLength() {
-      return maxLength;
-    }
   }
 
   // TODO: this if DocTermsIndex was already created, we

Modified: lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/TestDemoDocValue.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/TestDemoDocValue.java?rev=1430064&r1=1430063&r2=1430064&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/TestDemoDocValue.java
(original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/TestDemoDocValue.java
Mon Jan  7 22:29:33 2013
@@ -480,7 +480,6 @@ public class TestDemoDocValue extends Lu
     IndexReader ireader = DirectoryReader.open(directory); // read-only=true
     assert ireader.leaves().size() == 1;
     BinaryDocValues dv = ireader.leaves().get(0).reader().getBinaryDocValues("dv");
-    assertFalse(dv.isFixedLength());
     BytesRef scratch = new BytesRef();
     for(int i=0;i<2;i++) {
       StoredDocument doc2 = ireader.leaves().get(0).reader().document(i);

Modified: lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/index/TestDuelingCodecs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/index/TestDuelingCodecs.java?rev=1430064&r1=1430063&r2=1430064&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/index/TestDuelingCodecs.java
(original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/index/TestDuelingCodecs.java
Mon Jan  7 22:29:33 2013
@@ -636,8 +636,6 @@ public class TestDuelingCodecs extends L
         BinaryDocValues rightValues = MultiSimpleDocValues.simpleBinaryValues(rightReader,
field);
         if (leftValues != null && rightValues != null) {
           assertEquals(leftValues.size(), rightValues.size());
-          assertEquals(leftValues.maxLength(), rightValues.maxLength());
-          assertEquals(leftValues.isFixedLength(), rightValues.isFixedLength());
           BytesRef scratchLeft = new BytesRef();
           BytesRef scratchRight = new BytesRef();
           for(int docID=0;docID<leftValues.size();docID++) {

Modified: lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/search/TestFieldCache.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/search/TestFieldCache.java?rev=1430064&r1=1430063&r2=1430064&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/search/TestFieldCache.java
(original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/search/TestFieldCache.java
Mon Jan  7 22:29:33 2013
@@ -428,46 +428,4 @@ public class TestFieldCache extends Luce
     }
     assertFalse(failed.get());
   }
-
-  public void testMaxFixedLength() throws Exception {
-    Directory d = newDirectory();
-    RandomIndexWriter w = new RandomIndexWriter(random(), d, newIndexWriterConfig(TEST_VERSION_CURRENT,
new MockAnalyzer(random())));
-    int fixedLength = random().nextBoolean() ? -1 : _TestUtil.nextInt(random(), 1, 10);
-    int numTerms = atLeast(10);
-    int actualFixedLength = -2;
-    int actualMaxLength = -1;
-    for(int termIDX=0;termIDX<numTerms;termIDX++) {
-      Document doc = new Document();
-      String termString;
-      if (fixedLength == -1) {
-        termString = _TestUtil.randomRealisticUnicodeString(random());
-      } else {
-        termString = _TestUtil.randomRealisticUnicodeString(random(), fixedLength, fixedLength);
-      }
-      BytesRef term = new BytesRef(termString);
-      actualMaxLength = Math.max(actualMaxLength, term.length);
-      if (actualFixedLength == -2) {
-        actualFixedLength = term.length;
-      } else if (actualFixedLength != term.length) {
-        actualFixedLength = -1;
-      }
-      doc.add(newField("term", termString, StringField.TYPE_NOT_STORED));
-      w.addDocument(doc);
-    }
-    w.forceMerge(1);
-    IndexReader r = w.getReader();
-    w.close();
-    AtomicReader subR = r.leaves().get(0).reader();
-    BinaryDocValues values = FieldCache.DEFAULT.getTerms(subR, "term");
-    assertEquals(actualFixedLength >= 0, values.isFixedLength());
-    assertEquals(actualMaxLength, values.maxLength());
-
-    SortedDocValues sortedValues = FieldCache.DEFAULT.getTermsIndex(subR, "term");
-    assertEquals(actualFixedLength >= 0, sortedValues.isFixedLength());
-    assertEquals(actualMaxLength, sortedValues.maxLength());
-
-    // No field cache insanity because close purges FC entries:
-    r.close();
-    d.close();
-  }
 }

Modified: lucene/dev/branches/lucene4547/lucene/test-framework/src/java/org/apache/lucene/index/MultiSimpleDocValues.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/test-framework/src/java/org/apache/lucene/index/MultiSimpleDocValues.java?rev=1430064&r1=1430063&r2=1430064&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/test-framework/src/java/org/apache/lucene/index/MultiSimpleDocValues.java
(original)
+++ lucene/dev/branches/lucene4547/lucene/test-framework/src/java/org/apache/lucene/index/MultiSimpleDocValues.java
Mon Jan  7 22:29:33 2013
@@ -109,7 +109,6 @@ public class MultiSimpleDocValues {
   public static BinaryDocValues simpleBinaryValues(final IndexReader r, final String field)
throws IOException {
     final List<AtomicReaderContext> leaves = r.leaves();
     boolean anyReal = false;
-    int maxLength = -1;
 
     for(AtomicReaderContext ctx : leaves) {
       BinaryDocValues values = ctx.reader().getBinaryDocValues(field);
@@ -119,14 +118,11 @@ public class MultiSimpleDocValues {
       } else {
         anyReal = true;
       }
-      
-      maxLength = Math.max(maxLength, values.maxLength());
     }
 
     if (!anyReal) {
       return null;
     } else {
-      final int finalMaxLength = maxLength;
 
       return new BinaryDocValues() {
         @Override
@@ -150,17 +146,6 @@ public class MultiSimpleDocValues {
         public int size() {
           return r.maxDoc();
         }
-
-        @Override
-        public boolean isFixedLength() {
-          // Harmless lie?
-          return false;
-        }
-
-        @Override
-        public int maxLength() {
-          return finalMaxLength;
-        }
       };
     }
   }



Mime
View raw message