lucene-java-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mikemcc...@apache.org
Subject svn commit: r900882 - in /lucene/java/branches/flex_1458/src: java/org/apache/lucene/index/ java/org/apache/lucene/index/codecs/ java/org/apache/lucene/index/codecs/preflex/ java/org/apache/lucene/index/codecs/pulsing/ java/org/apache/lucene/index/code...
Date Tue, 19 Jan 2010 18:20:19 GMT
Author: mikemccand
Date: Tue Jan 19 18:20:17 2010
New Revision: 900882

URL: http://svn.apache.org/viewvc?rev=900882&view=rev
Log:
LUCENE-2111: use BytesRef for payloads in flex API; other improvements

Modified:
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DocsEnum.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/FreqProxTermsWriter.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/LegacyFieldsEnum.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/PositionsEnum.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/SegmentReader.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/PositionsConsumer.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/PreFlexFields.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/pulsing/PulsingDocsReader.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/pulsing/PulsingDocsWriter.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/sep/IntIndexOutput.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/sep/SepDocsReader.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/sep/SepPositionsReader.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/sep/SepPositionsWriter.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardCodec.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardDocsReader.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardPositionsReader.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardPositionsWriter.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardTermsDictReader.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/ExactPhraseScorer.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/MultiPhraseQuery.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/PhrasePositions.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/Similarity.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/payloads/PayloadTermQuery.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/spans/TermSpans.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/BytesRef.java
    lucene/java/branches/flex_1458/src/test/org/apache/lucene/TestExternalCodecs.java
    lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/FlexTestUtil.java
    lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
    lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestCodecs.java
    lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestOmitTf.java
    lucene/java/branches/flex_1458/src/test/org/apache/lucene/search/TestPhraseQuery.java

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DocsEnum.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DocsEnum.java?rev=900882&r1=900881&r2=900882&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DocsEnum.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DocsEnum.java Tue Jan 19 18:20:17 2010
@@ -65,12 +65,11 @@
     return count;
   }
 
-  // nocommit -- maybe move this up to TermsEnum?  that
-  // would disallow changing positions format/reader of each
-  // doc, though
-  // nocommit - doc whether this returns null if there are
-  // no positions, or a faker
   /** Don't call next() or skipTo() or read() until you're
-   *  done consuming the positions */
+   *  done consuming the positions.  NOTE: this method may
+   *  return null, if the index contains no positional
+   *  information for this document.  The standard codec
+   *  (default) does this today when the field was indexed
+   *  with {@link Field#setOmitTermFreqAndPositions}. */
   public abstract PositionsEnum positions() throws IOException;
 }

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/FreqProxTermsWriter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/FreqProxTermsWriter.java?rev=900882&r1=900881&r2=900882&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/FreqProxTermsWriter.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/FreqProxTermsWriter.java Tue Jan 19 18:20:17 2010
@@ -145,6 +145,7 @@
   }
 
   private byte[] payloadBuffer;
+  BytesRef payload;
 
   /* Walk through all unique text tokens (Posting
    * instances) found in this field and serialize them
@@ -244,19 +245,29 @@
             //System.out.println("    pos=" + position);
 
             final int payloadLength;
+            final BytesRef thisPayload;
+
             if ((code & 1) != 0) {
               // This position has a payload
-              payloadLength = prox.readVInt();
-
-              if (payloadBuffer == null || payloadBuffer.length < payloadLength)
-                payloadBuffer = new byte[payloadLength];
+              payloadLength = prox.readVInt();  
+              
+              if (payload == null) {
+                payload = new BytesRef();
+                payload.bytes = new byte[payloadLength];
+              } else if (payload.bytes.length < payloadLength) {
+                payload.grow(payloadLength);
+              }
+
+              prox.readBytes(payload.bytes, 0, payloadLength);
+              payload.length = payloadLength;
+              thisPayload = payload;
 
-              prox.readBytes(payloadBuffer, 0, payloadLength);
-
-            } else
+            } else {
               payloadLength = 0;
+              thisPayload = null;
+            }
 
-            posConsumer.addPosition(position, payloadBuffer, 0, payloadLength);
+            posConsumer.add(position, thisPayload);
           } //End for
 
           posConsumer.finishDoc();

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/LegacyFieldsEnum.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/LegacyFieldsEnum.java?rev=900882&r1=900881&r2=900882&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/LegacyFieldsEnum.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/LegacyFieldsEnum.java Tue Jan 19 18:20:17 2010
@@ -259,9 +259,23 @@
       return tp.getPayloadLength();
     }
 
+    private BytesRef payload;
+
     @Override
-    public byte[] getPayload(byte[] data, int offset) throws IOException {
-      return tp.getPayload(data, offset);
+    public BytesRef getPayload() throws IOException {
+      final int len = tp.getPayloadLength();
+      if (payload == null) {
+        payload = new BytesRef();
+        payload.bytes = new byte[len];
+      } else {
+        if (payload.bytes.length < len) {
+          payload.grow(len);
+        }
+      }
+      
+      payload.bytes = tp.getPayload(payload.bytes, 0);
+      payload.length = len;
+      return payload;
     }
 
     @Override

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/PositionsEnum.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/PositionsEnum.java?rev=900882&r1=900881&r2=900882&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/PositionsEnum.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/PositionsEnum.java Tue Jan 19 18:20:17 2010
@@ -20,6 +20,7 @@
 import java.io.IOException;
 
 import org.apache.lucene.util.AttributeSource;
+import org.apache.lucene.util.BytesRef;
 
 public abstract class PositionsEnum {
 
@@ -41,12 +42,12 @@
    *  the behavior is not defined. */
   public abstract int next() throws IOException;
 
+  /** Returns length of payload at current position */
   public abstract int getPayloadLength();
 
-  // nocommit -- how to use BytesRef here?
-  // nocommit -- improve this so that readers that do their
-  // own buffering can save a copy
-  public abstract byte[] getPayload(byte[] data, int offset) throws IOException;
+  /** Returns the payload at this position, or null if no
+   *  payload was indexed. */
+  public abstract BytesRef getPayload() throws IOException;
 
   public abstract boolean hasPayload();
 }

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/SegmentReader.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/SegmentReader.java?rev=900882&r1=900881&r2=900882&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/SegmentReader.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/SegmentReader.java Tue Jan 19 18:20:17 2010
@@ -927,9 +927,10 @@
       SegmentTermPositions stp = new SegmentTermPositions(pre.freqStream, pre.proxStream, pre.tis, core.fieldInfos);
       stp.setSkipDocs(deletedDocs);
       return stp;
-    } else
+    } else {
       // Emulate old API
       return new LegacyTermPositions();
+    }
   }
 
   @Override
@@ -1564,6 +1565,7 @@
   final private class LegacyTermPositions extends LegacyTermDocs implements TermPositions {
 
     PositionsEnum positions;
+    boolean didGetPositions;
 
     LegacyTermPositions() throws IOException {
       super();
@@ -1572,14 +1574,18 @@
     @Override
     public void seek(TermEnum termEnum) throws IOException {
       super.seek(termEnum);
-      if (docs != null)
+      if (docs != null) {
         positions = docs.positions();
+        didGetPositions = true;
+      } else {
+        didGetPositions = false;
+      }
     }
 
     @Override
     public boolean skipTo(int target) throws IOException {
       boolean result = super.skipTo(target);
-      positions = null;
+      didGetPositions = false;
       return result;
     }
 
@@ -1591,32 +1597,58 @@
     @Override
     public void seek(Term term) throws IOException {
       super.seek(term);
-      positions = null;
+      didGetPositions = false;
     }
 
     @Override
     public boolean next() throws IOException {
       boolean result = super.next();
-      positions = null;
+      didGetPositions = false;
       return result;
     }
 
     public int nextPosition() throws IOException {     
-      if (positions == null) {
+      if (!didGetPositions) {
         positions = docs.positions();
+        didGetPositions = true;
+      }
+        
+      if (positions == null) {
+        // With omitTFAP, pre-flex API pretended there was
+        // one occurrence of the term, at position 0:
+        return 0;
+      } else {
+        return positions.next();
       }
-      return positions.next();
     }
 
     public int getPayloadLength() {
+      if (positions == null) {
+        return 0;
+      }
       return positions.getPayloadLength();
     }
 
-    public byte[] getPayload(byte[] data, int offset) throws IOException {
-      return positions.getPayload(data, offset);
+    public byte[] getPayload(byte[] bytes, int offset) throws IOException {
+      final BytesRef payload = positions.getPayload();
+      // old API would always used passed in bytes if it
+      // "fits", else allocate new:
+      if (bytes != null && payload.length <= bytes.length - offset) {
+        System.arraycopy(payload.bytes, payload.offset, bytes, offset, payload.length);
+        return bytes;
+      } else if (payload.offset == 0 && payload.length == payload.bytes.length) {
+        return payload.bytes;
+      } else {
+        final byte[] retBytes = new byte[payload.length];
+        System.arraycopy(payload.bytes, payload.offset, retBytes, 0, payload.length);
+        return retBytes;
+      }
     }
 
     public boolean isPayloadAvailable() {
+      if (positions == null) {
+        return false;
+      }
       return positions.hasPayload();
     }
   }

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/PositionsConsumer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/PositionsConsumer.java?rev=900882&r1=900881&r2=900882&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/PositionsConsumer.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/PositionsConsumer.java Tue Jan 19 18:20:17 2010
@@ -20,20 +20,22 @@
 import java.io.IOException;
 
 import org.apache.lucene.index.PositionsEnum;
+import org.apache.lucene.util.BytesRef;
 
 public abstract class PositionsConsumer {
 
-  /** Add a new position & payload.  If payloadLength > 0
-   *  you must read those bytes from the IndexInput.  NOTE:
-   *  you must fully consume the byte[] payload, since
-   *  caller is free to reuse it on subsequent calls. */
-  public abstract void addPosition(int position, byte[] payload, int payloadOffset, int payloadLength) throws IOException;
+  /** Add a new position & payload.  A null payload means no
+   *  payload; a non-null payload with zero length also
+   *  means no payload.  Caller may reuse the {@link
+   *  BytesRef} for the payload between calls (method must
+   *  fully consume the payload). */
+  public abstract void add(int position, BytesRef payload) throws IOException;
 
   /** Called when we are done adding positions & payloads
    * for each doc */
   public abstract void finishDoc() throws IOException;
 
-  private byte[] payloadBuffer;
+  private BytesRef payload;
 
   /** Default merge impl, just copies positions & payloads
    *  from the input. */
@@ -41,13 +43,14 @@
     for(int i=0;i<freq;i++) {
       final int position = positions.next();
       final int payloadLength = positions.getPayloadLength();
+
+      final BytesRef payload;
       if (payloadLength > 0) {
-        if (payloadBuffer == null || payloadBuffer.length < payloadLength) {
-          payloadBuffer = new byte[payloadLength];
-        }
-        positions.getPayload(payloadBuffer, 0);
+        payload = positions.getPayload();
+      } else {
+        payload = null;
       }
-      addPosition(position, payloadBuffer, 0, payloadLength);
+      add(position, payload);
     }
     finishDoc();
   }

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/PreFlexFields.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/PreFlexFields.java?rev=900882&r1=900881&r2=900882&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/PreFlexFields.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/PreFlexFields.java Tue Jan 19 18:20:17 2010
@@ -439,9 +439,23 @@
       return pos.isPayloadAvailable();
     }
 
+    private BytesRef payload;
+
     @Override
-    public byte[] getPayload(byte[] data, int offset) throws IOException {
-      return pos.getPayload(data, offset);
+    public BytesRef getPayload() throws IOException {
+      final int len = pos.getPayloadLength();
+      if (payload == null) {
+        payload = new BytesRef();
+        payload.bytes = new byte[len];
+      } else {
+        if (payload.bytes.length < len) {
+          payload.grow(len);
+        }
+      }
+      
+      payload.bytes = pos.getPayload(payload.bytes, 0);
+      payload.length = len;
+      return payload;
     }
   }
 }

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/pulsing/PulsingDocsReader.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/pulsing/PulsingDocsReader.java?rev=900882&r1=900881&r2=900882&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/pulsing/PulsingDocsReader.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/pulsing/PulsingDocsReader.java Tue Jan 19 18:20:17 2010
@@ -31,6 +31,7 @@
 import org.apache.lucene.store.IndexInput;
 import org.apache.lucene.util.ArrayUtil;
 import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.BytesRef;
 
 /** Concrete class that reads the current doc/freq/skip
  *  postings format */
@@ -151,19 +152,26 @@
               final int code2 = termsIn.readVInt();
               if (storePayloads) {
                 position += code2 >>> 1;
-                if ((code2 & 1) != 0)
+                if ((code2 & 1) != 0) {
                   payloadLength = termsIn.readVInt();
+                }
+
                 if (payloadLength > 0) {
-                  if (pos.payload == null || payloadLength > pos.payload.length) {
-                    pos.payload = new byte[ArrayUtil.getNextSize(payloadLength)];
+                  if (pos.payload == null) {
+                    pos.payload = new BytesRef();
+                    pos.payload.bytes = new byte[payloadLength];
+                  } else if (payloadLength > pos.payload.bytes.length) {
+                    pos.payload.grow(payloadLength);
                   }
-                  termsIn.readBytes(pos.payload, 0, payloadLength);
+                  pos.payload.length = payloadLength;
+                  termsIn.readBytes(pos.payload.bytes, 0, payloadLength);
+                } else if (pos.payload != null) {
+                  pos.payload.length = 0;
                 }
               } else {
                 position += code2;
               }
               pos.pos = position;
-              pos.payloadLength = payloadLength;
             }
           }
           doc.docID = docID;
@@ -273,25 +281,17 @@
 
         @Override
         public int getPayloadLength() {
-          return pos.payloadLength;
+          return pos.payload == null ? 0 : pos.payload.length;
         }
 
         @Override
         public boolean hasPayload() {
-          // nocommit -- maybe don't do the payloadRetrieved check?
-          return !payloadRetrieved && pos.payloadLength > 0;
+          return pos.payload != null && pos.payload.length > 0;
         }
 
         @Override
-        public byte[] getPayload(byte[] data, int offset) {
-          // nocommit -- inefficient
-          if (!payloadRetrieved) {
-            payloadRetrieved = true;
-            System.arraycopy(pos.payload, 0, data, offset, pos.payloadLength);
-            return data;
-          } else {
-            return null;
-          }
+        public BytesRef getPayload() {
+          return pos.payload;
         }
       }
       

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/pulsing/PulsingDocsWriter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/pulsing/PulsingDocsWriter.java?rev=900882&r1=900881&r2=900882&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/pulsing/PulsingDocsWriter.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/pulsing/PulsingDocsWriter.java Tue Jan 19 18:20:17 2010
@@ -27,6 +27,7 @@
 import org.apache.lucene.index.codecs.standard.StandardPositionsConsumer;
 import org.apache.lucene.store.IndexOutput;
 import org.apache.lucene.util.ArrayUtil;
+import org.apache.lucene.util.BytesRef;
 
 // TODO: we now pulse entirely according to docFreq of the
 // term; it might be better to eg pulse by "net bytes used"
@@ -96,18 +97,15 @@
   boolean pulsed;                                 // false if we've seen > maxPulsingDocFreq docs
 
   static class Position {
-    byte[] payload;
+    BytesRef payload;
     int pos;
-    int payloadLength;
     
     @Override
     public Object clone() {
       Position position = new Position();
       position.pos = pos;
-      position.payloadLength = payloadLength;
-      if(payload != null) {
-        position.payload = new byte[payload.length];
-        System.arraycopy(payload, 0, position.payload, 0, payloadLength);
+      if (payload != null) {
+        position.payload = new BytesRef(payload);
       }
       return position;
     }
@@ -164,27 +162,33 @@
   class PositionsWriter extends StandardPositionsConsumer {
     @Override
     public void start(IndexOutput termsOut) {}
+
     @Override
     public void startTerm() {}
+
     @Override
-    public void addPosition(int position, byte[] payload, int payloadOffset, int payloadLength) {
+    public void add(int position, BytesRef payload) {
       Position pos = currentDoc.positions[currentDoc.numPositions++];
       pos.pos = position;
-      if (payload != null && payloadLength > 0) {
-        if (pos.payload == null || payloadLength > pos.payload.length) {
-          pos.payload = new byte[ArrayUtil.getNextSize(payloadLength)];
+      if (payload != null && payload.length > 0) {
+        if (pos.payload == null) {
+          pos.payload = new BytesRef(payload);
+        } else {
+          pos.payload.copy(payload);
         }
-        System.arraycopy(payload, payloadOffset, pos.payload, 0, payloadLength);
-        pos.payloadLength = payloadLength;
-      } else
-        pos.payloadLength = 0;
+      } else if (pos.payload != null) {
+        pos.payload.length = 0;
+      }
     }
+
     @Override
     public void finishDoc() {
       assert currentDoc.numPositions == currentDoc.termDocFreq;
     }
+
     @Override
     public void finishTerm(boolean isIndexTerm) {}
+
     @Override
     public void close() {}
   }
@@ -219,11 +223,12 @@
           assert doc.termDocFreq == doc.numPositions;
           for(int j=0;j<doc.termDocFreq;j++) {
             final Position pos = doc.positions[j];
-            if (pos.payload != null && pos.payloadLength > 0) {
+            if (pos.payload != null && pos.payload.length > 0) {
               assert storePayloads;
-              posConsumer.addPosition(pos.pos, pos.payload, 0, pos.payloadLength);
-            } else
-              posConsumer.addPosition(pos.pos, null, 0, 0);
+              posConsumer.add(pos.pos, pos.payload);
+            } else {
+              posConsumer.add(pos.pos, null);
+            }
           }
           posConsumer.finishDoc();
         }
@@ -305,16 +310,21 @@
             final int delta2 = pos.pos - lastPosition;
             lastPosition = pos.pos;
             if (storePayloads) {
-              if (pos.payloadLength != lastPayloadLength) {
+              final int payloadLength = pos.payload == null ? 0 : pos.payload.length;
+              if (payloadLength != lastPayloadLength) {
                 termsOut.writeVInt((delta2 << 1)|1);
-                termsOut.writeVInt(pos.payloadLength);
-                lastPayloadLength = pos.payloadLength;
-              } else
+                termsOut.writeVInt(payloadLength);
+                lastPayloadLength = payloadLength;
+              } else {
                 termsOut.writeVInt(delta2 << 1);
-              if (pos.payloadLength > 0)
-                termsOut.writeBytes(pos.payload, 0, pos.payloadLength);
-            } else
+              }
+
+              if (payloadLength > 0) {
+                termsOut.writeBytes(pos.payload.bytes, 0, pos.payload.length);
+              }
+            } else {
               termsOut.writeVInt(delta2);
+            }
           }
         }
       }

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/sep/IntIndexOutput.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/sep/IntIndexOutput.java?rev=900882&r1=900881&r2=900882&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/sep/IntIndexOutput.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/sep/IntIndexOutput.java Tue Jan 19 18:20:17 2010
@@ -60,4 +60,4 @@
 
   // nocommit
   public abstract String descFilePointer() throws IOException;
-}
\ No newline at end of file
+}

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/sep/SepDocsReader.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/sep/SepDocsReader.java?rev=900882&r1=900881&r2=900882&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/sep/SepDocsReader.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/sep/SepDocsReader.java Tue Jan 19 18:20:17 2010
@@ -398,19 +398,12 @@
           // Lazy init
           if (posReader == null) {
 
-            // nocommit -- should we return null?
-
             // TermFreq was omitted from this field during
             // indexing, which means we pretend termFreq is
             // always 1 with that 1 occurrence having
             // position 0
-            if (fakePositions == null) {
-              fakePositions = new FakePositionsEnum();
-            }
-            if (Codec.DEBUG) {
-              System.out.println("  return fake");
-            }
-            return fakePositions;
+            return null;
+
           } else {
 
             // nocommit: abstraction violation
@@ -609,24 +602,3 @@
     }
   }
 }
-
-/** Returned when someone asks for positions() enum on field
- *  with omitTf true */
-class FakePositionsEnum extends PositionsEnum {
-  @Override
-  public int next() {
-    return 0;
-  }
-  @Override
-  public int getPayloadLength() {
-    return 0;
-  }
-  @Override
-  public boolean hasPayload() {
-    return false;
-  }
-  @Override
-  public byte[] getPayload(byte[] data, int offset) {
-    return null;
-  }
-}

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/sep/SepPositionsReader.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/sep/SepPositionsReader.java?rev=900882&r1=900881&r2=900882&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/sep/SepPositionsReader.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/sep/SepPositionsReader.java Tue Jan 19 18:20:17 2010
@@ -28,6 +28,7 @@
 import org.apache.lucene.index.codecs.standard.StandardPositionsProducer;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.util.BytesRef;
 
 /** @lucene.experimental */
 public class SepPositionsReader extends StandardPositionsProducer {
@@ -281,32 +282,35 @@
         return payloadLength;
       }
 
+      private BytesRef payload;
+
       @Override
-      public byte[] getPayload(byte[] data, int offset) throws IOException {
+      public BytesRef getPayload() throws IOException {
 
         if (!payloadPending) {
           throw new IOException("Either no payload exists at this term position or an attempt was made to load it more than once.");
         }
 
+        if (payloadLength == 0) {
+          return null;
+        }
+
         if (Codec.DEBUG) {
           System.out.println("   getPayload payloadFP=" + payloadIn.getFilePointer() + " len=" + payloadLength);
         }
 
-        final byte[] retArray;
-        final int retOffset;
-        if (data == null || data.length-offset < payloadLength) {
-          // the array is too small to store the payload data,
-          // so we allocate a new one
-          retArray = new byte[payloadLength];
-          retOffset = 0;
-        } else {
-          retArray = data;
-          retOffset = offset;
+        if (payload == null) {
+          payload = new BytesRef();
+          payload.bytes = new byte[payloadLength];
+        } else if (payload.bytes.length < payloadLength) {
+          payload.grow(payloadLength);
         }
 
-        payloadIn.readBytes(retArray, retOffset, payloadLength);
+        payloadIn.readBytes(payload.bytes, 0, payloadLength);
         payloadPending = false;
-        return retArray;
+        payload.length = payloadLength;
+
+        return payload;
       }
       
       @Override

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/sep/SepPositionsWriter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/sep/SepPositionsWriter.java?rev=900882&r1=900881&r2=900882&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/sep/SepPositionsWriter.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/sep/SepPositionsWriter.java Tue Jan 19 18:20:17 2010
@@ -25,6 +25,7 @@
 import org.apache.lucene.index.IndexFileNames;
 import org.apache.lucene.index.codecs.PositionsConsumer;
 import org.apache.lucene.index.codecs.Codec;
+import org.apache.lucene.util.BytesRef;
 
 /** @lucene.experimental */
 public final class SepPositionsWriter extends PositionsConsumer {
@@ -113,12 +114,12 @@
 
   /** Add a new position & payload */
   @Override
-  public void addPosition(int position, byte[] payload, int payloadOffset, int payloadLength) throws IOException {
+  public void add(int position, BytesRef payload) throws IOException {
     assert !omitTF: "omitTF is true";
     assert posOut != null;
     if (Codec.DEBUG) {
-      if (payload != null) {
-        System.out.println("pw.addPos [" + desc + "]: pos=" + position + " posFP=" + posOut.descFilePointer() + " payloadFP=" + payloadOut.getFilePointer() + " payload=" + payloadLength + " bytes");
+      if (payload != null && payload.length > 0) {
+        System.out.println("pw.addPos [" + desc + "]: pos=" + position + " posFP=" + posOut.descFilePointer() + " payloadFP=" + payloadOut.getFilePointer() + " payload=" + payload.length + " bytes");
       } else {
         System.out.println("pw.addPos [" + desc + "]: pos=" + position + " posFP=" + posOut.descFilePointer() + " payloadFP=" + payloadOut.getFilePointer());
       }
@@ -128,6 +129,7 @@
     lastPosition = position;
 
     if (storePayloads) {
+      int payloadLength = payload == null ? 0 : payload.length;
       if (Codec.DEBUG) {
         System.out.println("  store payload len=" + payloadLength);
       }
@@ -148,7 +150,7 @@
         if (Codec.DEBUG) {
           System.out.println("  write @ payloadFP=" + payloadOut.getFilePointer());
         }
-        payloadOut.writeBytes(payload, payloadLength);
+        payloadOut.writeBytes(payload.bytes, payload.offset, payloadLength);
       }
     } else {
       posOut.write(delta);

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardCodec.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardCodec.java?rev=900882&r1=900881&r2=900882&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardCodec.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardCodec.java Tue Jan 19 18:20:17 2010
@@ -29,7 +29,7 @@
 import org.apache.lucene.index.codecs.FieldsProducer;
 import org.apache.lucene.store.Directory;
 
-/** Current index file format */
+/** Default codec. */
 public class StandardCodec extends Codec {
 
   public StandardCodec() {

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardDocsReader.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardDocsReader.java?rev=900882&r1=900881&r2=900882&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardDocsReader.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardDocsReader.java Tue Jan 19 18:20:17 2010
@@ -86,8 +86,9 @@
 
     skipInterval = termsIn.readInt();
     maxSkipLevels = termsIn.readInt();
-    if (posReader != null)
+    if (posReader != null) {
       posReader.start(termsIn);
+    }
   }
 
   @Override
@@ -405,10 +406,7 @@
             // indexing, which means we pretend termFreq is
             // always 1 with that 1 occurrence having
             // position 0
-            if (fakePositions == null) {
-              fakePositions = new FormatPostingsFakePositionsEnum();
-            }
-            return fakePositions;
+            return null;
           } else {
             // TODO: abstraction violation
             positions = (StandardPositionsReader.TermsDictReader.SegmentPositionsEnum) posReader.positions();
@@ -507,24 +505,3 @@
     }
   }
 }
-
-/** Returned when someone asks for positions() enum on field
- *  with omitTf true */
-class FormatPostingsFakePositionsEnum extends PositionsEnum {
-  @Override
-  public int next() {
-    return 0;
-  }
-  @Override
-  public int getPayloadLength() {
-    return 0;
-  }
-  @Override
-  public boolean hasPayload() {
-    return false;
-  }
-  @Override
-  public byte[] getPayload(byte[] data, int offset) {
-    return null;
-  }
-}

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardPositionsReader.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardPositionsReader.java?rev=900882&r1=900881&r2=900882&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardPositionsReader.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardPositionsReader.java Tue Jan 19 18:20:17 2010
@@ -27,9 +27,8 @@
 import org.apache.lucene.index.codecs.Codec;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.util.BytesRef;
 
-// nocommit -- base class should not be named terms dict:
-// this class interacts w/ a docsreader
 public class StandardPositionsReader extends StandardPositionsProducer {
   
   IndexInput proxIn;
@@ -120,8 +119,6 @@
       return positions;
     }
 
-      // nocommit -- should we have different reader for
-      // payload vs no payload?
     class SegmentPositionsEnum extends PositionsEnum {
 
       // nocommit
@@ -241,27 +238,24 @@
         return payloadLength;
       }
 
-      @Override
-      public byte[] getPayload(byte[] data, int offset) throws IOException {
+      private BytesRef payload;
 
-        if (!payloadPending)
+      @Override
+      public BytesRef getPayload() throws IOException {
+        if (!payloadPending) {
           throw new IOException("Either no payload exists at this term position or an attempt was made to load it more than once.");
-
-        final byte[] retArray;
-        final int retOffset;
-        if (data == null || data.length-offset < payloadLength) {
-          // the array is too small to store the payload data,
-          // so we allocate a new one
-          retArray = new byte[payloadLength];
-          retOffset = 0;
-        } else {
-          retArray = data;
-          retOffset = offset;
         }
-
-        proxIn.readBytes(retArray, retOffset, payloadLength);
+        if (payload == null) {
+          payload = new BytesRef();
+          payload.bytes = new byte[payloadLength];
+        } else if (payloadLength > payload.bytes.length) {
+          payload.grow(payloadLength);
+        }
+        proxIn.readBytes(payload.bytes, 0, payloadLength);
+        payload.length = payloadLength;
         payloadPending = false;
-        return retArray;
+
+        return payload;
       }
       
       @Override

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardPositionsWriter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardPositionsWriter.java?rev=900882&r1=900881&r2=900882&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardPositionsWriter.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardPositionsWriter.java Tue Jan 19 18:20:17 2010
@@ -24,6 +24,7 @@
 import org.apache.lucene.index.SegmentWriteState;
 import org.apache.lucene.index.codecs.Codec;
 import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.util.BytesRef;
 
 final class StandardPositionsWriter extends StandardPositionsConsumer {
   final static String CODEC = "SingleFilePositionsPayloads";
@@ -79,13 +80,13 @@
 
   /** Add a new position & payload */
   @Override
-  public void addPosition(int position, byte[] payload, int payloadOffset, int payloadLength) throws IOException {
+  public void add(int position, BytesRef payload) throws IOException {
     assert !omitTermFreqAndPositions: "omitTermFreqAndPositions is true";
     assert out != null;
 
     if (Codec.DEBUG) {
       if (payload != null)
-        System.out.println("pw.addPos [" + desc + "]: pos=" + position + " fp=" + out.getFilePointer() + " payload=" + payloadLength + " bytes");
+        System.out.println("pw.addPos [" + desc + "]: pos=" + position + " fp=" + out.getFilePointer() + " payload=" + payload.length + " bytes");
       else
         System.out.println("pw.addPos [" + desc + "]: pos=" + position + " fp=" + out.getFilePointer());
     }
@@ -100,6 +101,7 @@
       if (Codec.DEBUG) {
         System.out.println("  store payloads");
       }
+      final int payloadLength = payload == null ? 0 : payload.length;
 
       if (payloadLength != lastPayloadLength) {
         if (Codec.DEBUG) {
@@ -109,12 +111,16 @@
         lastPayloadLength = payloadLength;
         out.writeVInt((delta<<1)|1);
         out.writeVInt(payloadLength);
-      } else
+      } else {
         out.writeVInt(delta << 1);
-      if (payloadLength > 0)
-        out.writeBytes(payload, payloadLength);
-    } else
+      }
+
+      if (payloadLength > 0) {
+        out.writeBytes(payload.bytes, payload.offset, payloadLength);
+      }
+    } else {
       out.writeVInt(delta);
+    }
   }
 
   void setField(FieldInfo fieldInfo) {

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardTermsDictReader.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardTermsDictReader.java?rev=900882&r1=900881&r2=900882&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardTermsDictReader.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardTermsDictReader.java Tue Jan 19 18:20:17 2010
@@ -328,6 +328,7 @@
         CacheEntry entry = null;
         BytesRef entryKey = null;
 
+        // Consult terms cache first:
         if (docs.canCaptureState()) {
           entry = termsCache.get(term);
           if (entry != null) {
@@ -350,6 +351,7 @@
 
         if (termUpto != -1 && termUpto < numTerms) {
 
+          // See if we are already positioned at the requested term
           final int cmp = termComp.compare(bytesReader.term, term);
 
           if (cmp == 0) {

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/ExactPhraseScorer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/ExactPhraseScorer.java?rev=900882&r1=900881&r2=900882&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/ExactPhraseScorer.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/ExactPhraseScorer.java Tue Jan 19 18:20:17 2010
@@ -42,11 +42,11 @@
     int freq = 0;
     do {					  // find position w/ all terms
       while (first.position < last.position) {	  // scan forward in first
-	    do {
-	      if (!first.nextPosition())
-	        return freq;
-	    } while (first.position < last.position);
-	      firstToLast();
+        do {
+          if (!first.nextPosition())
+            return freq;
+        } while (first.position < last.position);
+        firstToLast();
       }
       freq++;					  // all equal: a match
     } while (last.nextPosition());

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/MultiPhraseQuery.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/MultiPhraseQuery.java?rev=900882&r1=900881&r2=900882&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/MultiPhraseQuery.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/MultiPhraseQuery.java Tue Jan 19 18:20:17 2010
@@ -526,7 +526,7 @@
     }
 
     @Override
-    public byte[] getPayload(byte[] data, int offset) {
+    public BytesRef getPayload() {
       throw new UnsupportedOperationException();
     }
 

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/PhrasePositions.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/PhrasePositions.java?rev=900882&r1=900881&r2=900882&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/PhrasePositions.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/PhrasePositions.java Tue Jan 19 18:20:17 2010
@@ -44,10 +44,6 @@
       return false;
     }
     positions = docs.positions();
-
-    // nocommit -- really needed?
-    //position = 0;
-
     return true;
   }
 
@@ -56,8 +52,6 @@
     if (doc == docs.NO_MORE_DOCS) {
       return false;
     }
-    // nocommit -- really needed?
-    // position = 0;
     return true;
   }
 
@@ -65,6 +59,9 @@
   final void firstPosition() throws IOException {
     count = docs.freq();				  // read first pos
     positions = docs.positions();
+    if (positions == null) {
+      throw new IllegalStateException("no positions are stored for this field (Field.omitTermFreqAndPositions was used)");
+    }
     nextPosition();
   }
 

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/Similarity.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/Similarity.java?rev=900882&r1=900881&r2=900882&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/Similarity.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/Similarity.java Tue Jan 19 18:20:17 2010
@@ -858,6 +858,7 @@
    * @return An implementation dependent float to be used as a scoring factor
    *
    */
+  // nocommit -- swtich to BytesRef
   public float scorePayload(int docId, String fieldName, int start, int end, byte [] payload, int offset, int length)
   {
     return 1;

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/payloads/PayloadTermQuery.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/payloads/PayloadTermQuery.java?rev=900882&r1=900881&r2=900882&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/payloads/PayloadTermQuery.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/payloads/PayloadTermQuery.java Tue Jan 19 18:20:17 2010
@@ -30,6 +30,7 @@
 import org.apache.lucene.search.spans.SpanTermQuery;
 import org.apache.lucene.search.spans.SpanWeight;
 import org.apache.lucene.search.spans.SpanScorer;
+import org.apache.lucene.util.BytesRef;
 
 import java.io.IOException;
 
@@ -80,8 +81,7 @@
     }
 
     protected class PayloadTermSpanScorer extends SpanScorer {
-      // TODO: is this the best way to allocate this?
-      protected byte[] payload = new byte[256];
+      protected BytesRef payload;
       protected float payloadScore;
       protected int payloadsSeen;
       private final TermSpans termSpans;
@@ -117,11 +117,22 @@
       protected void processPayload(Similarity similarity) throws IOException {
         final PositionsEnum positions = termSpans.getPositions();
         if (positions.hasPayload()) {
-          payload = positions.getPayload(payload, 0);
-          payloadScore = function.currentScore(doc, term.field(),
-              spans.start(), spans.end(), payloadsSeen, payloadScore,
-              similarity.scorePayload(doc, term.field(), spans.start(), spans
-                  .end(), payload, 0, positions.getPayloadLength()));
+          payload = positions.getPayload();
+          if (payload != null) {
+            payloadScore = function.currentScore(doc, term.field(),
+                                                 spans.start(), spans.end(), payloadsSeen, payloadScore,
+                                                 similarity.scorePayload(doc, term.field(), spans.start(),
+                                                                         spans.end(), payload.bytes,
+                                                                         payload.offset,
+                                                                         payload.length));
+          } else {
+            payloadScore = function.currentScore(doc, term.field(),
+                                                 spans.start(), spans.end(), payloadsSeen, payloadScore,
+                                                 similarity.scorePayload(doc, term.field(), spans.start(),
+                                                                         spans.end(), null,
+                                                                         0,
+                                                                         0));
+          }
           payloadsSeen++;
 
         } else {

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/spans/TermSpans.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/spans/TermSpans.java?rev=900882&r1=900881&r2=900882&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/spans/TermSpans.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/spans/TermSpans.java Tue Jan 19 18:20:17 2010
@@ -19,6 +19,7 @@
 import org.apache.lucene.index.Term;
 import org.apache.lucene.index.DocsEnum;
 import org.apache.lucene.index.PositionsEnum;
+import org.apache.lucene.util.BytesRef;
 
 import java.io.IOException;
 import java.util.Collections;
@@ -52,6 +53,9 @@
       }
       freq = docs.freq();
       positions = docs.positions();
+      if (positions == null) {
+        throw new IllegalStateException("no positions are stored for this field (Field.omitTermFreqAndPositions was used)");
+      }
       count = 0;
     }
     position = positions.next();
@@ -69,6 +73,9 @@
     freq = docs.freq();
     count = 0;
     positions = docs.positions();
+    if (positions == null) {
+      throw new IllegalStateException("no positions are stored for this field (Field.omitTermFreqAndPositions was used)");
+    }
 
     position = positions.next();
     count++;
@@ -94,8 +101,14 @@
   // TODO: Remove warning after API has been finalized
   @Override
   public Collection<byte[]> getPayload() throws IOException {
-    byte [] bytes = new byte[positions.getPayloadLength()]; 
-    bytes = positions.getPayload(bytes, 0);
+    final BytesRef payload = positions.getPayload();
+    final byte[] bytes;
+    if (payload != null) {
+      bytes = new byte[payload.length];
+      System.arraycopy(payload.bytes, payload.offset, bytes, 0, payload.length);
+    } else {
+      bytes = null;
+    }
     return Collections.singletonList(bytes);
   }
 

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/BytesRef.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/BytesRef.java?rev=900882&r1=900881&r2=900882&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/BytesRef.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/BytesRef.java Tue Jan 19 18:20:17 2010
@@ -32,6 +32,18 @@
   public BytesRef() {
   }
 
+  public BytesRef(byte[] bytes, int offset, int length) {
+    this.bytes = bytes;
+    this.offset = offset;
+    this.length = length;
+  }
+
+  public BytesRef(byte[] bytes) {
+    this.bytes = bytes;
+    this.offset = 0;
+    this.length = bytes.length;
+  }
+
   /**
    * @param text Initialize the byte[] from the UTF8 bytes
    * for the provided Sring.  This must be well-formed

Modified: lucene/java/branches/flex_1458/src/test/org/apache/lucene/TestExternalCodecs.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/test/org/apache/lucene/TestExternalCodecs.java?rev=900882&r1=900881&r2=900882&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/test/org/apache/lucene/TestExternalCodecs.java (original)
+++ lucene/java/branches/flex_1458/src/test/org/apache/lucene/TestExternalCodecs.java Tue Jan 19 18:20:17 2010
@@ -219,7 +219,7 @@
       }
 
       @Override
-      public void addPosition(int position, byte[] payload, int payloadOffset, int payloadLength) {
+      public void add(int position, BytesRef payload) {
         if (payload != null) {
           throw new UnsupportedOperationException("can't handle payloads");
         }
@@ -413,7 +413,7 @@
       }
 
       @Override
-      public byte[] getPayload(byte[] data, int offset) {
+      public BytesRef getPayload() {
         return null;
       }
     }

Modified: lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/FlexTestUtil.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/FlexTestUtil.java?rev=900882&r1=900881&r2=900882&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/FlexTestUtil.java (original)
+++ lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/FlexTestUtil.java Tue Jan 19 18:20:17 2010
@@ -158,18 +158,24 @@
               //System.out.println("TEST:     doc=" + doc + " freq=" + docs.freq());
               final int freq = docs.freq();
               PositionsEnum pos = docs.positions();
-              for(int i=0;i<freq;i++) {
-                final int position = pos.next();
-                //System.out.println("TEST:       pos=" + position);
-                assertEquals(position, termPos.nextPosition());
-                assertEquals(pos.hasPayload(), termPos.isPayloadAvailable());
-                if (pos.hasPayload()) {
-                  assertEquals(pos.getPayloadLength(), termPos.getPayloadLength());
-                  byte[] b1 = pos.getPayload(null, 0);
-                  byte[] b2 = termPos.getPayload(null, 0);
-                  assertNotNull(b1);
-                  assertNotNull(b2);
-                  assertTrue(Arrays.equals(b1, b2));
+              if (pos == null) {
+                assertEquals(1, freq);
+                assertEquals(0, termPos.nextPosition());
+                assertEquals(false, termPos.isPayloadAvailable());
+              } else {
+                for(int i=0;i<freq;i++) {
+                  final int position = pos.next();
+                  //System.out.println("TEST:       pos=" + position);
+                  assertEquals(position, termPos.nextPosition());
+                  assertEquals(pos.hasPayload(), termPos.isPayloadAvailable());
+                  if (pos.hasPayload()) {
+                    assertEquals(pos.getPayloadLength(), termPos.getPayloadLength());
+                    BytesRef payload = pos.getPayload();
+                    byte[] b2 = termPos.getPayload(null, 0);
+                    assertNotNull(payload);
+                    assertNotNull(b2);
+                    assertTrue(equals(payload, b2));
+                  }
                 }
               }
             }
@@ -257,19 +263,25 @@
                       // enum the positions
                       final int freq = docs.freq();
                       PositionsEnum pos = docs.positions();
-                      for(int i=0;i<freq;i++) {
-                        final int position = pos.next();
-                        //System.out.println("TEST:       pos=" + position);
-                        assertEquals(position, termPos.nextPosition());
-                        assertEquals(pos.hasPayload(), termPos.isPayloadAvailable());
-                        if (pos.hasPayload()) {
-                          assertEquals(pos.getPayloadLength(), termPos.getPayloadLength());
-                          if (rand.nextInt(3) <= 1) {
-                            byte[] b1 = pos.getPayload(null, 0);
-                            byte[] b2 = termPos.getPayload(null, 0);
-                            assertNotNull(b1);
-                            assertNotNull(b2);
-                            assertTrue(Arrays.equals(b1, b2));
+                      if (pos == null) {
+                        assertEquals(1, termPos.freq());
+                        assertEquals(0, termPos.nextPosition());
+                        assertFalse(termPos.isPayloadAvailable());
+                      } else {
+                        for(int i=0;i<freq;i++) {
+                          final int position = pos.next();
+                          //System.out.println("TEST:       pos=" + position);
+                          assertEquals(position, termPos.nextPosition());
+                          assertEquals(pos.hasPayload(), termPos.isPayloadAvailable());
+                          if (pos.hasPayload()) {
+                            assertEquals(pos.getPayloadLength(), termPos.getPayloadLength());
+                            if (rand.nextInt(3) <= 1) {
+                              BytesRef payload = pos.getPayload();
+                              byte[] b2 = termPos.getPayload(null, 0);
+                              assertNotNull(payload);
+                              assertNotNull(b2);
+                              assertTrue(equals(payload, b2));
+                            }
                           }
                         }
                       }
@@ -297,6 +309,20 @@
     // enum positions, sometimes skipping payloads
   }
 
+  private static boolean equals(BytesRef b1, byte[] b2) {
+    if (b1.length == b2.length) {
+      final int end = b1.offset + b1.length;
+      for(int i=b1.offset;i<end;i++) {
+        if (b1.bytes[i] != b2[i-b1.offset]) {
+          return false;
+        }
+      }
+      return true;
+    } else {
+      return false;
+    }
+  }
+
   public static int nextInt(Random rand, int min, int max) {
     return min + rand.nextInt(max-min);
   }
@@ -406,18 +432,24 @@
         doc = newDoc1;
 
         PositionsEnum posEnum = docs.positions();
-        for(int k=0;k<docs.freq();k++) {
-          int pos1 = posEnum.next();
-          int pos2 = termPositions.nextPosition();
-          assertEquals(pos1, pos2);
-          assertEquals(posEnum.hasPayload(), termPositions.isPayloadAvailable());
-          if (posEnum.hasPayload()) {
-            assertEquals(posEnum.getPayloadLength(), termPositions.getPayloadLength());
-            byte[] b1 = posEnum.getPayload(null, 0);
-            byte[] b2 = termPositions.getPayload(null, 0);
-            assertNotNull(b1);
-            assertNotNull(b2);
-            assertTrue(Arrays.equals(b1, b2));
+        if (posEnum == null) {
+          assertEquals(1, termPositions.freq());
+          assertEquals(0, termPositions.nextPosition());
+          assertFalse(termPositions.isPayloadAvailable());
+        } else {
+          for(int k=0;k<docs.freq();k++) {
+            int pos1 = posEnum.next();
+            int pos2 = termPositions.nextPosition();
+            assertEquals(pos1, pos2);
+            assertEquals(posEnum.hasPayload(), termPositions.isPayloadAvailable());
+            if (posEnum.hasPayload()) {
+              assertEquals(posEnum.getPayloadLength(), termPositions.getPayloadLength());
+              BytesRef b1 = posEnum.getPayload();
+              byte[] b2 = termPositions.getPayload(null, 0);
+              assertNotNull(b1);
+              assertNotNull(b2);
+              assertTrue(equals(b1, b2));
+            }
           }
         }
       }

Modified: lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java?rev=900882&r1=900881&r2=900882&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java (original)
+++ lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java Tue Jan 19 18:20:17 2010
@@ -67,11 +67,11 @@
   // oldNames array.
 
   /*
-  public void xxxtestCreatePreLocklessCFS() throws IOException {
+  public void testCreatePreLocklessCFS() throws IOException {
     createIndex("index.cfs", true);
   }
 
-  public void xxxtestCreatePreLocklessNoCFS() throws IOException {
+  public void testCreatePreLocklessNoCFS() throws IOException {
     createIndex("index.nocfs", false);
   }
   */
@@ -270,7 +270,7 @@
     }
   }
 
-  private void testHits(ScoreDoc[] hits, int expectedCount, IndexReader reader) throws IOException {
+  private void doTestHits(ScoreDoc[] hits, int expectedCount, IndexReader reader) throws IOException {
     final int hitCount = hits.length;
     assertEquals("wrong number of hits", expectedCount, hitCount);
     for(int i=0;i<hitCount;i++) {
@@ -331,7 +331,7 @@
     Document d = searcher.doc(hits[0].doc);
     assertEquals("didn't get the right document first", "21", d.get("id"));
 
-    testHits(hits, 34, searcher.getIndexReader());
+    doTestHits(hits, 34, searcher.getIndexReader());
 
     if (!oldName.startsWith("19.") &&
         !oldName.startsWith("20.") &&
@@ -387,7 +387,7 @@
     ScoreDoc[] hits = searcher.search(new TermQuery(new Term("content", "aaa")), null, 1000).scoreDocs;
     Document d = searcher.doc(hits[0].doc);
     assertEquals("wrong first document", "21", d.get("id"));
-    testHits(hits, 44, searcher.getIndexReader());
+    doTestHits(hits, 44, searcher.getIndexReader());
     searcher.close();
 
     // make sure we can do delete & setNorm against this
@@ -405,7 +405,7 @@
     assertEquals("wrong number of hits", 43, hits.length);
     d = searcher.doc(hits[0].doc);
     assertEquals("wrong first document", "22", d.get("id"));
-    testHits(hits, 43, searcher.getIndexReader());
+    doTestHits(hits, 43, searcher.getIndexReader());
     searcher.close();
 
     // optimize
@@ -417,7 +417,7 @@
     hits = searcher.search(new TermQuery(new Term("content", "aaa")), null, 1000).scoreDocs;
     assertEquals("wrong number of hits", 43, hits.length);
     d = searcher.doc(hits[0].doc);
-    testHits(hits, 43, searcher.getIndexReader());
+    doTestHits(hits, 43, searcher.getIndexReader());
     assertEquals("wrong first document", "22", d.get("id"));
     searcher.close();
 
@@ -455,7 +455,7 @@
     assertEquals("wrong number of hits", 33, hits.length);
     d = searcher.doc(hits[0].doc);
     assertEquals("wrong first document", "22", d.get("id"));
-    testHits(hits, 33, searcher.getIndexReader());
+    doTestHits(hits, 33, searcher.getIndexReader());
     searcher.close();
 
     // optimize
@@ -468,7 +468,7 @@
     assertEquals("wrong number of hits", 33, hits.length);
     d = searcher.doc(hits[0].doc);
     assertEquals("wrong first document", "22", d.get("id"));
-    testHits(hits, 33, searcher.getIndexReader());
+    doTestHits(hits, 33, searcher.getIndexReader());
     searcher.close();
 
     dir.close();

Modified: lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestCodecs.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestCodecs.java?rev=900882&r1=900881&r2=900882&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestCodecs.java (original)
+++ lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestCodecs.java Tue Jan 19 18:20:17 2010
@@ -130,9 +130,9 @@
 
   class PositionData {
     int pos;
-    byte[] payload;
+    BytesRef payload;
 
-    PositionData(int pos, byte[] payload) {
+    PositionData(int pos, BytesRef payload) {
       this.pos = pos;
       this.payload = payload;
     }
@@ -170,14 +170,12 @@
         if (!field.omitTF) {
           for(int j=0;j<positions[i].length;j++) {
             PositionData pos = positions[i][j];
-            if (pos.payload != null)
-              posConsumer.addPosition(pos.pos, pos.payload, 0, pos.payload.length);
-            else
-              posConsumer.addPosition(pos.pos, null, 0, 0);
+            posConsumer.add(pos.pos, pos.payload);
           }
           posConsumer.finishDoc();
-        } else
+        } else {
           assert posConsumer==null;
+        }
       }
       termsConsumer.finishTerm(text, docs.length);
     }
@@ -227,13 +225,16 @@
           for(int k=0;k<termFreq;k++) {
             position += nextInt(1, 10);
 
-            byte[] payload;
+            final BytesRef payload;
             if (storePayloads && nextInt(4) == 0) {
-              payload = new byte[1+nextInt(5)];
-              for(int l=0;l<payload.length;l++)
-                payload[l] = (byte) nextInt(255);
-            } else
+              byte[] bytes = new byte[1+nextInt(5)];
+              for(int l=0;l<bytes.length;l++) {
+                bytes[l] = (byte) nextInt(255);
+              }
+              payload = new BytesRef(bytes);
+            } else {
               payload = null;
+            }
 
             positions[j][k] = new PositionData(position, payload);
           }
@@ -377,17 +378,13 @@
         assertEquals(positions[i].pos, pos);
         if (positions[i].payload != null) {
           assertTrue(posEnum.hasPayload());
-          assertEquals(positions[i].payload.length, posEnum.getPayloadLength());
           if (nextInt(3) < 2) {
             if (Codec.DEBUG) {
               System.out.println("TEST do check payload len=" + posEnum.getPayloadLength());
             }
 
             // Verify the payload bytes
-            posEnum.getPayload(data, 0);
-            for(int j=0;j<positions[i].payload.length;j++) {
-              assertEquals(data[j], positions[i].payload[j]);
-            }
+            assertTrue(positions[i].payload.equals(posEnum.getPayload()));
           } else {
             if (Codec.DEBUG) {
               System.out.println("TEST skip check payload len=" + posEnum.getPayloadLength());

Modified: lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestOmitTf.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestOmitTf.java?rev=900882&r1=900881&r2=900882&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestOmitTf.java (original)
+++ lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestOmitTf.java Tue Jan 19 18:20:17 2010
@@ -27,13 +27,7 @@
 import org.apache.lucene.analysis.standard.StandardAnalyzer;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
-import org.apache.lucene.search.BooleanQuery;
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.Collector;
-import org.apache.lucene.search.Scorer;
-import org.apache.lucene.search.Searcher;
-import org.apache.lucene.search.Similarity;
-import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.*;
 import org.apache.lucene.search.BooleanClause.Occur;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.MockRAMDirectory;
@@ -298,6 +292,15 @@
     TermQuery q3 = new TermQuery(c);
     TermQuery q4 = new TermQuery(d);
 
+    PhraseQuery pq = new PhraseQuery();
+    pq.add(a);
+    pq.add(c);
+    try {
+      searcher.search(pq, 10);
+      fail("did not hit expected exception");
+    } catch (IllegalStateException ise) {
+      // expected
+    }
         
     searcher.search(q1,
                     new CountingHitCollector() {

Modified: lucene/java/branches/flex_1458/src/test/org/apache/lucene/search/TestPhraseQuery.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/test/org/apache/lucene/search/TestPhraseQuery.java?rev=900882&r1=900881&r2=900882&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/test/org/apache/lucene/search/TestPhraseQuery.java (original)
+++ lucene/java/branches/flex_1458/src/test/org/apache/lucene/search/TestPhraseQuery.java Tue Jan 19 18:20:17 2010
@@ -566,5 +566,5 @@
     q2.add(new PhraseQuery(), BooleanClause.Occur.MUST);
     q2.toString();
   }
-  
+
 }



Mime
View raw message