lucene-java-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From uschind...@apache.org
Subject svn commit: r1028739 - in /lucene/java/branches/lucene_3_0: ./ src/java/org/apache/lucene/analysis/standard/
Date Fri, 29 Oct 2010 13:44:30 GMT
Author: uschindler
Date: Fri Oct 29 13:44:30 2010
New Revision: 1028739

URL: http://svn.apache.org/viewvc?rev=1028739&view=rev
Log:
LUCENE-2384: Reset zzBuffer in StandardTokenizerImpl when lexer is reset

Modified:
    lucene/java/branches/lucene_3_0/CHANGES.txt
    lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java
    lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.java
    lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.jflex

Modified: lucene/java/branches/lucene_3_0/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_3_0/CHANGES.txt?rev=1028739&r1=1028738&r2=1028739&view=diff
==============================================================================
--- lucene/java/branches/lucene_3_0/CHANGES.txt (original)
+++ lucene/java/branches/lucene_3_0/CHANGES.txt Fri Oct 29 13:44:30 2010
@@ -48,6 +48,9 @@ Bug fixes
 * LUCENE-2650: Added extra safety to MMapIndexInput clones to prevent accessing
   an unmapped buffer if the input is closed (Mike McCandless, Uwe Schindler, Robert Muir)

   
+* LUCENE-2384: Reset zzBuffer in StandardTokenizerImpl when lexer is reset.
+  (Ruben Laguna via Uwe Schindler, sub-issue of LUCENE-2074) 
+  
 Documentation
 
 * LUCENE-2239: Documented limitations in NIOFSDirectory and MMapDirectory due 

Modified: lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java?rev=1028739&r1=1028738&r2=1028739&view=diff
==============================================================================
--- lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java
(original)
+++ lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java
Fri Oct 29 13:44:30 2010
@@ -203,21 +203,10 @@ public final class StandardTokenizer ext
     offsetAtt.setOffset(finalOffset, finalOffset);
   }
 
-  /*
-   * (non-Javadoc)
-   *
-   * @see org.apache.lucene.analysis.TokenStream#reset()
-   */
-  @Override
-  public void reset() throws IOException {
-    super.reset();
-    scanner.yyreset(input);
-  }
-
   @Override
   public void reset(Reader reader) throws IOException {
     super.reset(reader);
-    reset();
+    scanner.reset(reader);
   }
 
   /**

Modified: lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.java?rev=1028739&r1=1028738&r2=1028739&view=diff
==============================================================================
--- lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.java
(original)
+++ lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.java
Fri Oct 29 13:44:30 2010
@@ -364,6 +364,17 @@ public final int yychar()
 }
 
 /**
+ * Resets the Tokenizer to a new Reader.
+ */
+final void reset(java.io.Reader r) {
+  // reset to default buffer size, if buffer has grown
+  if (zzBuffer.length > ZZ_BUFFERSIZE) {
+    zzBuffer = new char[ZZ_BUFFERSIZE];
+  }
+  yyreset(r);
+}
+
+/**
  * Fills Lucene token with the current token text.
  */
 final void getText(Token t) {

Modified: lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.jflex
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.jflex?rev=1028739&r1=1028738&r2=1028739&view=diff
==============================================================================
--- lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.jflex
(original)
+++ lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.jflex
Fri Oct 29 13:44:30 2010
@@ -65,6 +65,17 @@ public final int yychar()
 }
 
 /**
+ * Resets the Tokenizer to a new Reader.
+ */
+final void reset(java.io.Reader r) {
+  // reset to default buffer size, if buffer has grown
+  if (zzBuffer.length > ZZ_BUFFERSIZE) {
+    zzBuffer = new char[ZZ_BUFFERSIZE];
+  }
+  yyreset(r);
+}
+
+/**
  * Fills Lucene token with the current token text.
  */
 final void getText(Token t) {



Mime
View raw message