lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From rm...@apache.org
Subject lucene-solr:branch_6x: LUCENE-7419: Don't lookup PositionIncrementAttribute every time in TokenStream.end()
Date Fri, 19 Aug 2016 15:26:21 GMT
Repository: lucene-solr
Updated Branches:
  refs/heads/branch_6x 082f36e01 -> 326db813f


LUCENE-7419: Don't lookup PositionIncrementAttribute every time in TokenStream.end()


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/326db813
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/326db813
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/326db813

Branch: refs/heads/branch_6x
Commit: 326db813fa1791a3ada5d5dafebfb47268f03632
Parents: 082f36e
Author: Robert Muir <rmuir@apache.org>
Authored: Fri Aug 19 11:14:08 2016 -0400
Committer: Robert Muir <rmuir@apache.org>
Committed: Fri Aug 19 11:15:21 2016 -0400

----------------------------------------------------------------------
 lucene/CHANGES.txt                                       |  3 +++
 .../src/java/org/apache/lucene/analysis/TokenStream.java |  7 +------
 .../tokenattributes/PackedTokenAttributeImpl.java        | 11 +++++++++++
 .../tokenattributes/PositionIncrementAttributeImpl.java  |  5 +++++
 .../src/java/org/apache/lucene/util/AttributeImpl.java   | 11 +++++++++++
 .../src/java/org/apache/lucene/util/AttributeSource.java | 10 ++++++++++
 6 files changed, 41 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/326db813/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 4cd8f3c..4e4078d 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -74,6 +74,9 @@ Bug Fixes
 * SOLR-9413: Fix analysis/kuromoji's CSVUtil.quoteEscape logic, add TestCSVUtil test.
   (AppChecker, Christine Poerschke)
 
+* LUCENE-7419: Fix performance bug with TokenStream.end(), where it would lookup
+  PositionIncrementAttribute every time. (Mike McCandless, Robert Muir)
+
 Improvements
 
 * LUCENE-7323: Compound file writing now verifies the incoming

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/326db813/lucene/core/src/java/org/apache/lucene/analysis/TokenStream.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/analysis/TokenStream.java b/lucene/core/src/java/org/apache/lucene/analysis/TokenStream.java
index 6a78e1c..af1e7bd 100644
--- a/lucene/core/src/java/org/apache/lucene/analysis/TokenStream.java
+++ b/lucene/core/src/java/org/apache/lucene/analysis/TokenStream.java
@@ -22,7 +22,6 @@ import java.io.Closeable;
 import java.lang.reflect.Modifier;
 
 import org.apache.lucene.analysis.tokenattributes.PackedTokenAttributeImpl;
-import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.index.IndexWriter;
@@ -176,11 +175,7 @@ public abstract class TokenStream extends AttributeSource implements
Closeable {
    * @throws IOException If an I/O error occurs
    */
   public void end() throws IOException {
-    clearAttributes(); // LUCENE-3849: don't consume dirty atts
-    PositionIncrementAttribute posIncAtt = getAttribute(PositionIncrementAttribute.class);
-    if (posIncAtt != null) {
-      posIncAtt.setPositionIncrement(0);
-    }
+    endAttributes(); // LUCENE-3849: don't consume dirty atts
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/326db813/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PackedTokenAttributeImpl.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PackedTokenAttributeImpl.java
b/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PackedTokenAttributeImpl.java
index a84d7b7..aaa3316 100644
--- a/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PackedTokenAttributeImpl.java
+++ b/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PackedTokenAttributeImpl.java
@@ -138,6 +138,17 @@ public class PackedTokenAttributeImpl extends CharTermAttributeImpl
     startOffset = endOffset = 0;
     type = DEFAULT_TYPE;
   }
+  
+  /** Resets the attributes at end
+   */
+  @Override
+  public void end() {
+    super.end();
+    positionIncrement = 0;
+    positionLength = 1;
+    startOffset = endOffset = 0;
+    type = DEFAULT_TYPE;
+  }
 
   @Override
   public PackedTokenAttributeImpl clone() {

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/326db813/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PositionIncrementAttributeImpl.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PositionIncrementAttributeImpl.java
b/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PositionIncrementAttributeImpl.java
index 283f481..4d63d6f 100644
--- a/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PositionIncrementAttributeImpl.java
+++ b/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PositionIncrementAttributeImpl.java
@@ -47,6 +47,11 @@ public class PositionIncrementAttributeImpl extends AttributeImpl implements
Pos
   }
   
   @Override
+  public void end() {
+    this.positionIncrement = 0;
+  }
+
+  @Override
   public boolean equals(Object other) {
     if (other == this) {
       return true;

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/326db813/lucene/core/src/java/org/apache/lucene/util/AttributeImpl.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/util/AttributeImpl.java b/lucene/core/src/java/org/apache/lucene/util/AttributeImpl.java
index b722661..8a7c542 100644
--- a/lucene/core/src/java/org/apache/lucene/util/AttributeImpl.java
+++ b/lucene/core/src/java/org/apache/lucene/util/AttributeImpl.java
@@ -33,6 +33,17 @@ public abstract class AttributeImpl implements Cloneable, Attribute {
   public abstract void clear();
   
   /**
+   * Clears the values in this AttributeImpl and resets it to its value
+   * at the end of the field. If this implementation implements more than one Attribute interface
+   * it clears all.
+   * <p>
+   * The default implementation simply calls {@link #clear()}
+   */
+  public void end() {
+    clear();
+  }
+  
+  /**
    * This method returns the current attribute values as a string in the following format
    * by calling the {@link #reflectWith(AttributeReflector)} method:
    * 

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/326db813/lucene/core/src/java/org/apache/lucene/util/AttributeSource.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/util/AttributeSource.java b/lucene/core/src/java/org/apache/lucene/util/AttributeSource.java
index 6c39025..e962fed 100644
--- a/lucene/core/src/java/org/apache/lucene/util/AttributeSource.java
+++ b/lucene/core/src/java/org/apache/lucene/util/AttributeSource.java
@@ -270,6 +270,16 @@ public class AttributeSource {
       state.attribute.clear();
     }
   }
+  
+  /**
+   * Resets all Attributes in this AttributeSource by calling
+   * {@link AttributeImpl#end()} on each Attribute implementation.
+   */
+  public final void endAttributes() {
+    for (State state = getCurrentState(); state != null; state = state.next) {
+      state.attribute.end();
+    }
+  }
 
   /**
    * Removes all attributes and their implementations from this AttributeSource.


Mime
View raw message