lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sh...@apache.org
Subject svn commit: r1433930 - in /lucene/dev/branches/lucene_solr_4_1: ./ lucene/ lucene/facet/ lucene/facet/src/java/org/apache/lucene/util/encoding/ lucene/facet/src/test/org/apache/lucene/facet/index/params/ lucene/facet/src/test/org/apache/lucene/util/enc...
Date Wed, 16 Jan 2013 13:28:58 GMT
Author: shaie
Date: Wed Jan 16 13:28:58 2013
New Revision: 1433930

URL: http://svn.apache.org/viewvc?rev=1433930&view=rev
Log:
LUCENE-4620: inline encoding/decoding

Removed:
    lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/VInt8.java
    lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/test/org/apache/lucene/util/encoding/Vint8Test.java
Modified:
    lucene/dev/branches/lucene_solr_4_1/   (props changed)
    lucene/dev/branches/lucene_solr_4_1/lucene/   (props changed)
    lucene/dev/branches/lucene_solr_4_1/lucene/facet/   (props changed)
    lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/ChunksIntEncoder.java
    lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/DGapIntDecoder.java
    lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/DGapIntEncoder.java
    lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/EightFlagsIntDecoder.java
    lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/EightFlagsIntEncoder.java
    lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/FourFlagsIntDecoder.java
    lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/FourFlagsIntEncoder.java
    lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/IntDecoder.java
    lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/IntEncoder.java
    lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/IntEncoderFilter.java
    lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/NOnesIntDecoder.java
    lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/NOnesIntEncoder.java
    lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/SimpleIntDecoder.java
    lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/SimpleIntEncoder.java
    lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/SortingIntEncoder.java
    lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/UniqueValuesIntEncoder.java
    lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/VInt8IntDecoder.java
    lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/VInt8IntEncoder.java
    lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/test/org/apache/lucene/facet/index/params/CategoryListParamsTest.java
    lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/test/org/apache/lucene/util/encoding/EncodingTest.java

Modified: lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/ChunksIntEncoder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/ChunksIntEncoder.java?rev=1433930&r1=1433929&r2=1433930&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/ChunksIntEncoder.java
(original)
+++ lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/ChunksIntEncoder.java
Wed Jan 16 13:28:58 2013
@@ -49,20 +49,20 @@ import org.apache.lucene.util.IntsRef;
  * @lucene.experimental
  */
 public abstract class ChunksIntEncoder extends IntEncoder {
-
+  
   /** Holds the values which must be encoded, outside the indicator. */
   protected final IntsRef encodeQueue;
-
+  
   /** Represents bits flag byte. */
   protected int indicator = 0;
-
+  
   /** Counts the current ordinal of the encoded value. */
   protected byte ordinal = 0;
-
+  
   protected ChunksIntEncoder(int chunkSize) {
     encodeQueue = new IntsRef(chunkSize);
   }
-
+  
   /**
    * Encodes the values of the current chunk. First it writes the indicator, and
    * then it encodes the values outside the indicator.
@@ -76,17 +76,40 @@ public abstract class ChunksIntEncoder e
     
     buf.bytes[buf.length++] = ((byte) indicator);
     for (int i = 0; i < encodeQueue.length; i++) {
-      VInt8.encode(encodeQueue.ints[i], buf);
+      // it is better if the encoding is inlined like so, and not e.g.
+      // in a utility method
+      int value = encodeQueue.ints[i];
+      if ((value & ~0x7F) == 0) {
+        buf.bytes[buf.length] = (byte) value;
+        buf.length++;
+      } else if ((value & ~0x3FFF) == 0) {
+        buf.bytes[buf.length] = (byte) (0x80 | ((value & 0x3F80) >> 7));
+        buf.bytes[buf.length + 1] = (byte) (value & 0x7F);
+        buf.length += 2;
+      } else if ((value & ~0x1FFFFF) == 0) {
+        buf.bytes[buf.length] = (byte) (0x80 | ((value & 0x1FC000) >> 14));
+        buf.bytes[buf.length + 1] = (byte) (0x80 | ((value & 0x3F80) >> 7));
+        buf.bytes[buf.length + 2] = (byte) (value & 0x7F);
+        buf.length += 3;
+      } else if ((value & ~0xFFFFFFF) == 0) {
+        buf.bytes[buf.length] = (byte) (0x80 | ((value & 0xFE00000) >> 21));
+        buf.bytes[buf.length + 1] = (byte) (0x80 | ((value & 0x1FC000) >> 14));
+        buf.bytes[buf.length + 2] = (byte) (0x80 | ((value & 0x3F80) >> 7));
+        buf.bytes[buf.length + 3] = (byte) (value & 0x7F);
+        buf.length += 4;
+      } else {
+        buf.bytes[buf.length] = (byte) (0x80 | ((value & 0xF0000000) >> 28));
+        buf.bytes[buf.length + 1] = (byte) (0x80 | ((value & 0xFE00000) >> 21));
+        buf.bytes[buf.length + 2] = (byte) (0x80 | ((value & 0x1FC000) >> 14));
+        buf.bytes[buf.length + 3] = (byte) (0x80 | ((value & 0x3F80) >> 7));
+        buf.bytes[buf.length + 4] = (byte) (value & 0x7F);
+        buf.length += 5;
+      }
     }
     
-    reset();
-  }
-
-  @Override
-  protected void reset() {
     ordinal = 0;
     indicator = 0;
     encodeQueue.length = 0;
   }
-
+  
 }

Modified: lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/DGapIntDecoder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/DGapIntDecoder.java?rev=1433930&r1=1433929&r2=1433930&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/DGapIntDecoder.java
(original)
+++ lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/DGapIntDecoder.java
Wed Jan 16 13:28:58 2013
@@ -26,7 +26,7 @@ import org.apache.lucene.util.IntsRef;
  * 
  * @lucene.experimental
  */
-public class DGapIntDecoder extends IntDecoder {
+public final class DGapIntDecoder extends IntDecoder {
 
   private final IntDecoder decoder;
 
@@ -35,13 +35,8 @@ public class DGapIntDecoder extends IntD
   }
 
   @Override
-  protected void reset() {
-    decoder.reset();
-  }
-  
-  @Override
-  protected void doDecode(BytesRef buf, IntsRef values, int upto) {
-    decoder.doDecode(buf, values, upto);
+  public void decode(BytesRef buf, IntsRef values) {
+    decoder.decode(buf, values);
     int prev = 0;
     for (int i = 0; i < values.length; i++) {
       values.ints[i] += prev;
@@ -51,7 +46,7 @@ public class DGapIntDecoder extends IntD
 
   @Override
   public String toString() {
-    return "DGap (" + decoder.toString() + ")";
+    return "DGap(" + decoder.toString() + ")";
   }
 
 }

Modified: lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/DGapIntEncoder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/DGapIntEncoder.java?rev=1433930&r1=1433929&r2=1433930&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/DGapIntEncoder.java
(original)
+++ lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/DGapIntEncoder.java
Wed Jan 16 13:28:58 2013
@@ -35,7 +35,7 @@ import org.apache.lucene.util.IntsRef;
  * 
  * @lucene.experimental
  */
-public class DGapIntEncoder extends IntEncoderFilter {
+public final class DGapIntEncoder extends IntEncoderFilter {
 
   /** Initializes with the given encoder. */
   public DGapIntEncoder(IntEncoder encoder) {
@@ -43,14 +43,15 @@ public class DGapIntEncoder extends IntE
   }
 
   @Override
-  protected void doEncode(IntsRef values, BytesRef buf, int upto) {
+  public void encode(IntsRef values, BytesRef buf) {
     int prev = 0;
+    int upto = values.offset + values.length;
     for (int i = values.offset; i < upto; i++) {
       int tmp = values.ints[i];
       values.ints[i] -= prev;
       prev = tmp;
     }
-    encoder.doEncode(values, buf, upto);
+    encoder.encode(values, buf);
   }
 
   @Override
@@ -60,7 +61,7 @@ public class DGapIntEncoder extends IntE
   
   @Override
   public String toString() {
-    return "DGap (" + encoder.toString() + ")";
+    return "DGap(" + encoder.toString() + ")";
   }
   
 }

Modified: lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/EightFlagsIntDecoder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/EightFlagsIntDecoder.java?rev=1433930&r1=1433929&r2=1433930&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/EightFlagsIntDecoder.java
(original)
+++ lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/EightFlagsIntDecoder.java
Wed Jan 16 13:28:58 2013
@@ -45,10 +45,13 @@ public class EightFlagsIntDecoder extend
   }
 
   @Override
-  protected void doDecode(BytesRef buf, IntsRef values, int upto) {
-    while (buf.offset < upto) {
+  public void decode(BytesRef buf, IntsRef values) {
+    values.offset = values.length = 0;
+    int upto = buf.offset + buf.length;
+    int offset = buf.offset;
+    while (offset < upto) {
       // read indicator
-      int indicator = buf.bytes[buf.offset++] & 0xFF;
+      int indicator = buf.bytes[offset++] & 0xFF;
       int ordinal = 0;
 
       int capacityNeeded = values.length + 8;
@@ -59,11 +62,21 @@ public class EightFlagsIntDecoder extend
       // process indicator, until we read 8 values, or end-of-buffer
       while (ordinal != 8) {
         if (DECODE_TABLE[indicator][ordinal++] == 0) {
-          if (buf.offset == upto) { // end of buffer
+          if (offset == upto) { // end of buffer
             return;
           }
-          // decode the value from the stream.
-          values.ints[values.length++] = VInt8.decode(buf) + 2; 
+          // it is better if the decoding is inlined like so, and not e.g.
+          // in a utility method
+          int value = 0;
+          while (true) {
+            byte b = buf.bytes[offset++];
+            if (b >= 0) {
+              values.ints[values.length++] = ((value << 7) | b) + 2;
+              break;
+            } else {
+              value = (value << 7) | (b & 0x7F);
+            }
+          }
         } else {
           values.ints[values.length++] = 1;
         }
@@ -73,7 +86,7 @@ public class EightFlagsIntDecoder extend
 
   @Override
   public String toString() {
-    return "EightFlags (VInt8)";
+    return "EightFlags(VInt8)";
   }
 
 }

Modified: lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/EightFlagsIntEncoder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/EightFlagsIntEncoder.java?rev=1433930&r1=1433929&r2=1433930&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/EightFlagsIntEncoder.java
(original)
+++ lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/EightFlagsIntEncoder.java
Wed Jan 16 13:28:58 2013
@@ -59,7 +59,9 @@ public class EightFlagsIntEncoder extend
   }
 
   @Override
-  protected void doEncode(IntsRef values, BytesRef buf, int upto) {
+  public void encode(IntsRef values, BytesRef buf) {
+    buf.offset = buf.length = 0;
+    int upto = values.offset + values.length;
     for (int i = values.offset; i < upto; i++) {
       int value = values.ints[i];
       if (value == 1) {
@@ -88,7 +90,7 @@ public class EightFlagsIntEncoder extend
 
   @Override
   public String toString() {
-    return "EightFlags (VInt)";
+    return "EightFlags(VInt)";
   }
 
 }

Modified: lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/FourFlagsIntDecoder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/FourFlagsIntDecoder.java?rev=1433930&r1=1433929&r2=1433930&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/FourFlagsIntDecoder.java
(original)
+++ lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/FourFlagsIntDecoder.java
Wed Jan 16 13:28:58 2013
@@ -45,10 +45,13 @@ public class FourFlagsIntDecoder extends
   }
 
   @Override
-  protected void doDecode(BytesRef buf, IntsRef values, int upto) {
-    while (buf.offset < upto) {
+  public void decode(BytesRef buf, IntsRef values) {
+    values.offset = values.length = 0;
+    int upto = buf.offset + buf.length;
+    int offset = buf.offset;
+    while (offset < upto) {
       // read indicator
-      int indicator = buf.bytes[buf.offset++] & 0xFF;
+      int indicator = buf.bytes[offset++] & 0xFF;
       int ordinal = 0;
       
       int capacityNeeded = values.length + 4;
@@ -59,11 +62,21 @@ public class FourFlagsIntDecoder extends
       while (ordinal != 4) {
         byte decodeVal = DECODE_TABLE[indicator][ordinal++];
         if (decodeVal == 0) {
-          if (buf.offset == upto) { // end of buffer
+          if (offset == upto) { // end of buffer
             return;
           }
-          // decode the value from the stream.
-          values.ints[values.length++] = VInt8.decode(buf) + 4;
+          // it is better if the decoding is inlined like so, and not e.g.
+          // in a utility method
+          int value = 0;
+          while (true) {
+            byte b = buf.bytes[offset++];
+            if (b >= 0) {
+              values.ints[values.length++] = ((value << 7) | b) + 4;
+              break;
+            } else {
+              value = (value << 7) | (b & 0x7F);
+            }
+          }
         } else {
           values.ints[values.length++] = decodeVal;
         }
@@ -73,7 +86,7 @@ public class FourFlagsIntDecoder extends
 
   @Override
   public String toString() {
-    return "FourFlags (VInt8)";
+    return "FourFlags(VInt8)";
   }
 
 }

Modified: lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/FourFlagsIntEncoder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/FourFlagsIntEncoder.java?rev=1433930&r1=1433929&r2=1433930&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/FourFlagsIntEncoder.java
(original)
+++ lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/FourFlagsIntEncoder.java
Wed Jan 16 13:28:58 2013
@@ -65,7 +65,9 @@ public class FourFlagsIntEncoder extends
   }
 
   @Override
-  protected void doEncode(IntsRef values, BytesRef buf, int upto) {
+  public void encode(IntsRef values, BytesRef buf) {
+    buf.offset = buf.length = 0;
+    int upto = values.offset + values.length;
     for (int i = values.offset; i < upto; i++) {
       int value = values.ints[i];
       if (value <= 3) {
@@ -94,7 +96,7 @@ public class FourFlagsIntEncoder extends
 
   @Override
   public String toString() {
-    return "FourFlags (VInt)";
+    return "FourFlags(VInt)";
   }
 
 }

Modified: lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/IntDecoder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/IntDecoder.java?rev=1433930&r1=1433929&r2=1433930&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/IntDecoder.java
(original)
+++ lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/IntDecoder.java
Wed Jan 16 13:28:58 2013
@@ -28,43 +28,9 @@ import org.apache.lucene.util.IntsRef;
 public abstract class IntDecoder {
   
   /**
-   * Performs the actual decoding. Values should be read from
-   * {@link BytesRef#offset} up to {@code upto}. Also, {@code values} offset and
-   * length are set to 0 and the encoder is expected to update
-   * {@link IntsRef#length}, but not {@link IntsRef#offset}.
-   * 
-   * <p>
-   * <b>NOTE:</b> it is ok to use the buffer's offset as the current position
in
-   * the buffer (and modify it), it will be reset by
-   * {@link #decode(BytesRef, IntsRef)}.
-   */
-  protected abstract void doDecode(BytesRef buf, IntsRef values, int upto);
-  
-  /**
-   * Called before {@link #doDecode(BytesRef, IntsRef, int)} so that decoders
-   * can reset their state.
-   */
-  protected void reset() {
-    // do nothing by default
-  }
-  
-  /**
    * Decodes the values from the buffer into the given {@link IntsRef}. Note
    * that {@code values.offset} and {@code values.length} are set to 0.
    */
-  public final void decode(BytesRef buf, IntsRef values) {
-    values.offset = values.length = 0; // must do that because we cannot grow() them otherwise
-    
-    // some decoders may use the buffer's offset as a position index, so save
-    // current offset.
-    int bufOffset = buf.offset;
-    
-    reset();
-    doDecode(buf, values, buf.offset + buf.length);
-    assert values.offset == 0 : "offset should not have been modified by the decoder.";
-    
-    // fix offset
-    buf.offset = bufOffset;
-  }
+  public abstract void decode(BytesRef buf, IntsRef values);
 
 }

Modified: lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/IntEncoder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/IntEncoder.java?rev=1433930&r1=1433929&r2=1433930&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/IntEncoder.java
(original)
+++ lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/IntEncoder.java
Wed Jan 16 13:28:58 2013
@@ -32,31 +32,10 @@ public abstract class IntEncoder {
   public IntEncoder() {}
 
   /**
-   * Performs the actual encoding. Values should be read from
-   * {@link IntsRef#offset} up to {@code upto}. Also, it is guaranteed that
-   * {@code buf's} offset and length are set to 0 and the encoder is expected to
-   * update {@link BytesRef#length}, but not {@link BytesRef#offset}.
-   */
-  protected abstract void doEncode(IntsRef values, BytesRef buf, int upto);
-  
-  /**
-   * Called before {@link #doEncode(IntsRef, BytesRef, int)} so that encoders
-   * can reset their state.
-   */
-  protected void reset() {
-    // do nothing by default
-  }
-
-  /**
    * Encodes the values to the given buffer. Note that the buffer's offset and
    * length are set to 0.
    */
-  public final void encode(IntsRef values, BytesRef buf) {
-    buf.offset = buf.length = 0;
-    reset();
-    doEncode(values, buf, values.offset + values.length);
-    assert buf.offset == 0 : "offset should not have been modified by the encoder.";
-  }
+  public abstract void encode(IntsRef values, BytesRef buf);
 
   /**
    * Returns an {@link IntDecoder} which can decode the values that were encoded

Modified: lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/IntEncoderFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/IntEncoderFilter.java?rev=1433930&r1=1433929&r2=1433930&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/IntEncoderFilter.java
(original)
+++ lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/IntEncoderFilter.java
Wed Jan 16 13:28:58 2013
@@ -31,9 +31,4 @@ public abstract class IntEncoderFilter e
     this.encoder = encoder;
   }
 
-  @Override
-  public void reset() {
-    encoder.reset();
-  }
-
 }

Modified: lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/NOnesIntDecoder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/NOnesIntDecoder.java?rev=1433930&r1=1433929&r2=1433930&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/NOnesIntDecoder.java
(original)
+++ lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/NOnesIntDecoder.java
Wed Jan 16 13:28:58 2013
@@ -42,14 +42,10 @@ public class NOnesIntDecoder extends Fou
   }
 
   @Override
-  protected void reset() {
+  public void decode(BytesRef buf, IntsRef values) {
+    values.offset = values.length = 0;
     internalBuffer.length = 0;
-    super.reset();
-  }
-  
-  @Override
-  protected void doDecode(BytesRef buf, IntsRef values, int upto) {
-    super.doDecode(buf, internalBuffer, upto);
+    super.decode(buf, internalBuffer);
     if (values.ints.length < internalBuffer.length) {
       // need space for internalBuffer.length to internalBuffer.length*N,
       // grow mildly at first
@@ -84,7 +80,7 @@ public class NOnesIntDecoder extends Fou
 
   @Override
   public String toString() {
-    return "NOnes (" + n + ") (" + super.toString() + ")";
+    return "NOnes(" + n + ") (" + super.toString() + ")";
   }
 
 }

Modified: lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/NOnesIntEncoder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/NOnesIntEncoder.java?rev=1433930&r1=1433929&r2=1433930&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/NOnesIntEncoder.java
(original)
+++ lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/NOnesIntEncoder.java
Wed Jan 16 13:28:58 2013
@@ -65,19 +65,15 @@ public class NOnesIntEncoder extends Fou
   }
 
   @Override
-  protected void reset() {
+  public void encode(IntsRef values, BytesRef buf) {
     internalBuffer.length = 0;
-    super.reset();
-  }
-  
-  @Override
-  protected void doEncode(IntsRef values, BytesRef buf, int upto) {
     // make sure the internal buffer is large enough
     if (values.length > internalBuffer.ints.length) {
       internalBuffer.grow(values.length);
     }
     
     int onesCounter = 0;
+    int upto = values.offset + values.length;
     for (int i = values.offset; i < upto; i++) {
       int value = values.ints[i];
       if (value == 1) {
@@ -102,7 +98,7 @@ public class NOnesIntEncoder extends Fou
       --onesCounter;
       internalBuffer.ints[internalBuffer.length++] = 1;
     }
-    super.doEncode(internalBuffer, buf, internalBuffer.length);
+    super.encode(internalBuffer, buf);
   }
 
   @Override
@@ -112,7 +108,7 @@ public class NOnesIntEncoder extends Fou
 
   @Override
   public String toString() {
-    return "NOnes (" + n + ") (" + super.toString() + ")";
+    return "NOnes(" + n + ") (" + super.toString() + ")";
   }
 
 }

Modified: lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/SimpleIntDecoder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/SimpleIntDecoder.java?rev=1433930&r1=1433929&r2=1433930&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/SimpleIntDecoder.java
(original)
+++ lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/SimpleIntDecoder.java
Wed Jan 16 13:28:58 2013
@@ -1,7 +1,9 @@
 package org.apache.lucene.util.encoding;
 
+import org.apache.lucene.util.ArrayUtil;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.IntsRef;
+import org.apache.lucene.util.RamUsageEstimator;
 
 /*
  * Licensed to the Apache Software Foundation (ASF) under one or more
@@ -25,19 +27,24 @@ import org.apache.lucene.util.IntsRef;
  * 
  * @lucene.experimental
  */
-public class SimpleIntDecoder extends IntDecoder {
+public final class SimpleIntDecoder extends IntDecoder {
 
   @Override
-  protected void doDecode(BytesRef buf, IntsRef values, int upto) {
-    while (buf.offset < upto) {
-      if (values.length == values.ints.length) {
-        values.grow(values.length + 10); // grow by few items, however not too many
-      }
+  public void decode(BytesRef buf, IntsRef values) {
+    values.offset = values.length = 0;
+    int numValues = buf.length / 4; // every value is 4 bytes
+    if (values.ints.length < numValues) { // offset and length are 0
+      values.ints = new int[ArrayUtil.oversize(numValues, RamUsageEstimator.NUM_BYTES_INT)];
+    }
+    
+    int offset = buf.offset;
+    int upto = buf.offset + buf.length;
+    while (offset < upto) {
       values.ints[values.length++] = 
-          ((buf.bytes[buf.offset++] & 0xFF) << 24) | 
-          ((buf.bytes[buf.offset++] & 0xFF) << 16) | 
-          ((buf.bytes[buf.offset++] & 0xFF) <<  8) | 
-          (buf.bytes[buf.offset++] & 0xFF);
+          ((buf.bytes[offset++] & 0xFF) << 24) | 
+          ((buf.bytes[offset++] & 0xFF) << 16) | 
+          ((buf.bytes[offset++] & 0xFF) <<  8) | 
+          (buf.bytes[offset++] & 0xFF);
     }
   }
 

Modified: lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/SimpleIntEncoder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/SimpleIntEncoder.java?rev=1433930&r1=1433929&r2=1433930&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/SimpleIntEncoder.java
(original)
+++ lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/SimpleIntEncoder.java
Wed Jan 16 13:28:58 2013
@@ -25,16 +25,18 @@ import org.apache.lucene.util.IntsRef;
  * 
  * @lucene.experimental
  */
-public class SimpleIntEncoder extends IntEncoder {
+public final class SimpleIntEncoder extends IntEncoder {
 
   @Override
-  protected void doEncode(IntsRef values, BytesRef buf, int upto) {
+  public void encode(IntsRef values, BytesRef buf) {
+    buf.offset = buf.length = 0;
     // ensure there's enough room in the buffer
     int bytesNeeded = values.length * 4;
     if (buf.bytes.length < bytesNeeded) {
       buf.grow(bytesNeeded);
     }
     
+    int upto = values.offset + values.length;
     for (int i = values.offset; i < upto; i++) {
       int value = values.ints[i];
       buf.bytes[buf.length++] = (byte) (value >>> 24);

Modified: lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/SortingIntEncoder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/SortingIntEncoder.java?rev=1433930&r1=1433929&r2=1433930&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/SortingIntEncoder.java
(original)
+++ lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/SortingIntEncoder.java
Wed Jan 16 13:28:58 2013
@@ -28,7 +28,7 @@ import org.apache.lucene.util.IntsRef;
  * 
  * @lucene.experimental
  */
-public class SortingIntEncoder extends IntEncoderFilter {
+public final class SortingIntEncoder extends IntEncoderFilter {
 
   /** Initializes with the given encoder. */
   public SortingIntEncoder(IntEncoder encoder) {
@@ -36,9 +36,9 @@ public class SortingIntEncoder extends I
   }
 
   @Override
-  protected void doEncode(IntsRef values, BytesRef buf, int upto) {
-    Arrays.sort(values.ints, values.offset, upto);
-    encoder.doEncode(values, buf, upto);
+  public void encode(IntsRef values, BytesRef buf) {
+    Arrays.sort(values.ints, values.offset, values.offset + values.length);
+    encoder.encode(values, buf);
   }
 
   @Override
@@ -48,7 +48,7 @@ public class SortingIntEncoder extends I
   
   @Override
   public String toString() {
-    return "Sorting (" + encoder.toString() + ")";
+    return "Sorting(" + encoder.toString() + ")";
   }
   
 }

Modified: lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/UniqueValuesIntEncoder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/UniqueValuesIntEncoder.java?rev=1433930&r1=1433929&r2=1433930&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/UniqueValuesIntEncoder.java
(original)
+++ lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/UniqueValuesIntEncoder.java
Wed Jan 16 13:28:58 2013
@@ -36,9 +36,10 @@ public final class UniqueValuesIntEncode
   }
 
   @Override
-  protected void doEncode(IntsRef values, BytesRef buf, int upto) {
+  public void encode(IntsRef values, BytesRef buf) {
     int prev = values.ints[values.offset];
     int idx = values.offset + 1;
+    int upto = values.offset + values.length;
     for (int i = idx; i < upto; i++) {
       if (values.ints[i] != prev) {
         values.ints[idx++] = values.ints[i];
@@ -46,7 +47,7 @@ public final class UniqueValuesIntEncode
       }
     }
     values.length = idx - values.offset;
-    encoder.doEncode(values, buf, idx);
+    encoder.encode(values, buf);
   }
 
   @Override
@@ -56,7 +57,7 @@ public final class UniqueValuesIntEncode
   
   @Override
   public String toString() {
-    return "Unique (" + encoder.toString() + ")";
+    return "Unique(" + encoder.toString() + ")";
   }
   
 }

Modified: lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/VInt8IntDecoder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/VInt8IntDecoder.java?rev=1433930&r1=1433929&r2=1433930&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/VInt8IntDecoder.java
(original)
+++ lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/VInt8IntDecoder.java
Wed Jan 16 13:28:58 2013
@@ -1,7 +1,9 @@
 package org.apache.lucene.util.encoding;
 
+import org.apache.lucene.util.ArrayUtil;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.IntsRef;
+import org.apache.lucene.util.RamUsageEstimator;
 
 /*
  * Licensed to the Apache Software Foundation (ASF) under one or more
@@ -25,15 +27,32 @@ import org.apache.lucene.util.IntsRef;
  * 
  * @lucene.experimental
  */
-public class VInt8IntDecoder extends IntDecoder {
+public final class VInt8IntDecoder extends IntDecoder {
 
   @Override
-  protected void doDecode(BytesRef buf, IntsRef values, int upto) {
-    while (buf.offset < upto) {
-      if (values.length == values.ints.length) {
-        values.grow(values.length + 10); // grow by few items, however not too many
+  public void decode(BytesRef buf, IntsRef values) {
+    values.offset = values.length = 0;
+
+    // grow the buffer up front, even if by a large number of values (buf.length)
+    // that saves the need to check inside the loop for every decoded value if
+    // the buffer needs to grow.
+    if (values.ints.length < buf.length) {
+      values.ints = new int[ArrayUtil.oversize(buf.length, RamUsageEstimator.NUM_BYTES_INT)];
+    }
+
+    // it is better if the decoding is inlined like so, and not e.g.
+    // in a utility method
+    int upto = buf.offset + buf.length;
+    int value = 0;
+    int offset = buf.offset;
+    while (offset < upto) {
+      byte b = buf.bytes[offset++];
+      if (b >= 0) {
+        values.ints[values.length++] = (value << 7) | b;
+        value = 0;
+      } else {
+        value = (value << 7) | (b & 0x7F);
       }
-      values.ints[values.length++] = VInt8.decode(buf);
     }
   }
 

Modified: lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/VInt8IntEncoder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/VInt8IntEncoder.java?rev=1433930&r1=1433929&r2=1433930&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/VInt8IntEncoder.java
(original)
+++ lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/VInt8IntEncoder.java
Wed Jan 16 13:28:58 2013
@@ -47,17 +47,47 @@ import org.apache.lucene.util.IntsRef;
  * 
  * @lucene.experimental
  */
-public class VInt8IntEncoder extends IntEncoder {
+public final class VInt8IntEncoder extends IntEncoder {
 
   @Override
-  protected void doEncode(IntsRef values, BytesRef buf, int upto) {
+  public void encode(IntsRef values, BytesRef buf) {
+    buf.offset = buf.length = 0;
     int maxBytesNeeded = 5 * values.length; // at most 5 bytes per VInt
     if (buf.bytes.length < maxBytesNeeded) {
       buf.grow(maxBytesNeeded);
     }
     
+    int upto = values.offset + values.length;
     for (int i = values.offset; i < upto; i++) {
-      VInt8.encode(values.ints[i], buf);
+      // it is better if the encoding is inlined like so, and not e.g.
+      // in a utility method
+      int value = values.ints[i];
+      if ((value & ~0x7F) == 0) {
+        buf.bytes[buf.length] = (byte) value;
+        buf.length++;
+      } else if ((value & ~0x3FFF) == 0) {
+        buf.bytes[buf.length] = (byte) (0x80 | ((value & 0x3F80) >> 7));
+        buf.bytes[buf.length + 1] = (byte) (value & 0x7F);
+        buf.length += 2;
+      } else if ((value & ~0x1FFFFF) == 0) {
+        buf.bytes[buf.length] = (byte) (0x80 | ((value & 0x1FC000) >> 14));
+        buf.bytes[buf.length + 1] = (byte) (0x80 | ((value & 0x3F80) >> 7));
+        buf.bytes[buf.length + 2] = (byte) (value & 0x7F);
+        buf.length += 3;
+      } else if ((value & ~0xFFFFFFF) == 0) {
+        buf.bytes[buf.length] = (byte) (0x80 | ((value & 0xFE00000) >> 21));
+        buf.bytes[buf.length + 1] = (byte) (0x80 | ((value & 0x1FC000) >> 14));
+        buf.bytes[buf.length + 2] = (byte) (0x80 | ((value & 0x3F80) >> 7));
+        buf.bytes[buf.length + 3] = (byte) (value & 0x7F);
+        buf.length += 4;
+      } else {
+        buf.bytes[buf.length] = (byte) (0x80 | ((value & 0xF0000000) >> 28));
+        buf.bytes[buf.length + 1] = (byte) (0x80 | ((value & 0xFE00000) >> 21));
+        buf.bytes[buf.length + 2] = (byte) (0x80 | ((value & 0x1FC000) >> 14));
+        buf.bytes[buf.length + 3] = (byte) (0x80 | ((value & 0x3F80) >> 7));
+        buf.bytes[buf.length + 4] = (byte) (value & 0x7F);
+        buf.length += 5;
+      }
     }
   }
 

Modified: lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/test/org/apache/lucene/facet/index/params/CategoryListParamsTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/test/org/apache/lucene/facet/index/params/CategoryListParamsTest.java?rev=1433930&r1=1433929&r2=1433930&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/test/org/apache/lucene/facet/index/params/CategoryListParamsTest.java
(original)
+++ lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/test/org/apache/lucene/facet/index/params/CategoryListParamsTest.java
Wed Jan 16 13:28:58 2013
@@ -1,6 +1,12 @@
 package org.apache.lucene.facet.index.params;
 
 import org.apache.lucene.index.Term;
+import org.apache.lucene.util.encoding.DGapIntEncoder;
+import org.apache.lucene.util.encoding.IntDecoder;
+import org.apache.lucene.util.encoding.IntEncoder;
+import org.apache.lucene.util.encoding.SortingIntEncoder;
+import org.apache.lucene.util.encoding.UniqueValuesIntEncoder;
+import org.apache.lucene.util.encoding.VInt8IntEncoder;
 import org.junit.Test;
 
 import org.apache.lucene.util.LuceneTestCase;
@@ -29,8 +35,10 @@ public class CategoryListParamsTest exte
   public void testDefaultSettings() {
     CategoryListParams clp = new CategoryListParams();
     assertEquals("wrong default term", new Term("$facets", "$fulltree$"), clp.getTerm());
-    assertEquals("unexpected default encoder", "Sorting (Unique (DGap (VInt8)))", clp.createEncoder().toString());
-    assertEquals("unexpected default decoder", "DGap (VInt8)", clp.createEncoder().createMatchingDecoder().toString());
+    IntEncoder encoder = new SortingIntEncoder(new UniqueValuesIntEncoder(new DGapIntEncoder(new
VInt8IntEncoder())));
+    IntDecoder decoder = encoder.createMatchingDecoder();
+    assertEquals("unexpected default encoder", encoder.toString(), clp.createEncoder().toString());
+    assertEquals("unexpected default decoder", decoder.toString(), clp.createEncoder().createMatchingDecoder().toString());
   }
   
   /**

Modified: lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/test/org/apache/lucene/util/encoding/EncodingTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/test/org/apache/lucene/util/encoding/EncodingTest.java?rev=1433930&r1=1433929&r2=1433930&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/test/org/apache/lucene/util/encoding/EncodingTest.java
(original)
+++ lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/test/org/apache/lucene/util/encoding/EncodingTest.java
Wed Jan 16 13:28:58 2013
@@ -64,9 +64,12 @@ public class EncodingTest extends Lucene
     
     BytesRef bytes = new BytesRef(100); // some initial capacity - encoders should grow the
byte[]
     IntsRef values = new IntsRef(100); // some initial capacity - decoders should grow the
int[]
-    encoding(encoder, data, bytes);
-    decoding(bytes, values, encoder.createMatchingDecoder());
-    assertTrue(expected.intsEquals(values));
+    for (int i = 0; i < 2; i++) {
+      // run 2 iterations to catch encoders/decoders which don't reset properly
+      encoding(encoder, data, bytes);
+      decoding(bytes, values, encoder.createMatchingDecoder());
+      assertTrue(expected.intsEquals(values));
+    }
   }
 
   private static void encoding(IntEncoder encoder, IntsRef data, BytesRef bytes) throws IOException
{



Mime
View raw message