lucene-java-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mikemcc...@apache.org
Subject svn commit: r787792 - in /lucene/java/branches/lucene_2_4_back_compat_tests/src/java/org/apache/lucene: analysis/CharReader.java analysis/Tokenizer.java analysis/standard/StandardTokenizer.java search/CharStream.java
Date Tue, 23 Jun 2009 19:09:02 GMT
Author: mikemccand
Date: Tue Jun 23 19:09:01 2009
New Revision: 787792

URL: http://svn.apache.org/viewvc?rev=787792&view=rev
Log:
LUCENE-1466: added chainable CharFilter stage before Tokenizer to allow mapping of characters
before tokenization

Added:
    lucene/java/branches/lucene_2_4_back_compat_tests/src/java/org/apache/lucene/analysis/CharReader.java
  (with props)
    lucene/java/branches/lucene_2_4_back_compat_tests/src/java/org/apache/lucene/search/CharStream.java
  (with props)
Modified:
    lucene/java/branches/lucene_2_4_back_compat_tests/src/java/org/apache/lucene/analysis/Tokenizer.java
    lucene/java/branches/lucene_2_4_back_compat_tests/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java

Added: lucene/java/branches/lucene_2_4_back_compat_tests/src/java/org/apache/lucene/analysis/CharReader.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_4_back_compat_tests/src/java/org/apache/lucene/analysis/CharReader.java?rev=787792&view=auto
==============================================================================
--- lucene/java/branches/lucene_2_4_back_compat_tests/src/java/org/apache/lucene/analysis/CharReader.java
(added)
+++ lucene/java/branches/lucene_2_4_back_compat_tests/src/java/org/apache/lucene/analysis/CharReader.java
Tue Jun 23 19:09:01 2009
@@ -0,0 +1,53 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.analysis;
+
+import java.io.IOException;
+import java.io.Reader;
+
+/**
+ * CharReader is a Reader wrapper. It reads chars from Reader and outputs CharStream.
+ *
+ * @version $Id$
+ *
+ */
+public final class CharReader extends CharStream {
+
+  protected Reader input;
+  
+  public static CharStream get(Reader input) {
+    return input instanceof CharStream ?
+      (CharStream)input : new CharReader(input);
+  }
+
+  private CharReader(Reader in) {
+    input = in;
+  }
+
+  public int correctOffset(int currentOff) {
+    return currentOff;
+  }
+
+  public void close() throws IOException {
+    input.close();
+  }
+
+  public int read(char[] cbuf, int off, int len) throws IOException {
+    return input.read(cbuf, off, len);
+  }
+}

Propchange: lucene/java/branches/lucene_2_4_back_compat_tests/src/java/org/apache/lucene/analysis/CharReader.java
------------------------------------------------------------------------------
    svn:eol-style = native

Modified: lucene/java/branches/lucene_2_4_back_compat_tests/src/java/org/apache/lucene/analysis/Tokenizer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_4_back_compat_tests/src/java/org/apache/lucene/analysis/Tokenizer.java?rev=787792&r1=787791&r2=787792&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_4_back_compat_tests/src/java/org/apache/lucene/analysis/Tokenizer.java
(original)
+++ lucene/java/branches/lucene_2_4_back_compat_tests/src/java/org/apache/lucene/analysis/Tokenizer.java
Tue Jun 23 19:09:01 2009
@@ -45,14 +45,14 @@
 
 public abstract class Tokenizer extends TokenStream {
   /** The text source for this Tokenizer. */
-  protected Reader input;
+  protected CharStream input;
 
   /** Construct a tokenizer with null input. */
   protected Tokenizer() {}
 
   /** Construct a token stream processing the given input. */
   protected Tokenizer(Reader input) {
-    this.input = input;
+    this.input = CharReader.get(input);
   }
 
   /** By default, closes the input Reader. */
@@ -64,7 +64,7 @@
    *  analyzer (in its reusableTokenStream method) will use
    *  this to re-use a previously created tokenizer. */
   public void reset(Reader input) throws IOException {
-    this.input = input;
+    this.input = CharReader.get(input);
   }
 }
 

Modified: lucene/java/branches/lucene_2_4_back_compat_tests/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_4_back_compat_tests/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java?rev=787792&r1=787791&r2=787792&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_4_back_compat_tests/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java
(original)
+++ lucene/java/branches/lucene_2_4_back_compat_tests/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java
Tue Jun 23 19:09:01 2009
@@ -26,6 +26,8 @@
 import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
 import org.apache.lucene.analysis.tokenattributes.TermAttribute;
 import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
+import org.apache.lucene.analysis.CharReader;
+import org.apache.lucene.analysis.CharStream;
 
 /** A grammar-based tokenizer constructed with JFlex
  *
@@ -91,7 +93,7 @@
   private boolean replaceInvalidAcronym;
     
   void setInput(Reader reader) {
-    this.input = reader;
+    this.input = CharReader.get(reader);
   }
 
   private int maxTokenLength = StandardAnalyzer.DEFAULT_MAX_TOKEN_LENGTH;
@@ -126,7 +128,7 @@
    */
   public StandardTokenizer(Reader input, boolean replaceInvalidAcronym) {
     this.replaceInvalidAcronym = replaceInvalidAcronym;
-    this.input = input;
+    setInput(input);
     this.scanner = new StandardTokenizerImpl(input);
     termAtt = (TermAttribute) addAttribute(TermAttribute.class);
     offsetAtt = (OffsetAttribute) addAttribute(OffsetAttribute.class);
@@ -240,7 +242,7 @@
     }
 
     public void reset(Reader reader) throws IOException {
-        input = reader;
+        setInput(reader);
         reset();
     }
 

Added: lucene/java/branches/lucene_2_4_back_compat_tests/src/java/org/apache/lucene/search/CharStream.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_4_back_compat_tests/src/java/org/apache/lucene/search/CharStream.java?rev=787792&view=auto
==============================================================================
--- lucene/java/branches/lucene_2_4_back_compat_tests/src/java/org/apache/lucene/search/CharStream.java
(added)
+++ lucene/java/branches/lucene_2_4_back_compat_tests/src/java/org/apache/lucene/search/CharStream.java
Tue Jun 23 19:09:01 2009
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.analysis;
+
+import java.io.Reader;
+
+/**
+ * CharStream adds <a href="#correctOffset(int)">correctOffset</a> functionality
over Reader.
+ *
+ * @version $Id$
+ *
+ */
+public abstract class CharStream extends Reader {
+
+  /**
+   * called by CharFilter(s) and Tokenizer to correct token offset.
+   *
+   * @param currentOff current offset
+   * @return corrected token offset
+   */
+  public abstract int correctOffset( int currentOff );
+}

Propchange: lucene/java/branches/lucene_2_4_back_compat_tests/src/java/org/apache/lucene/search/CharStream.java
------------------------------------------------------------------------------
    svn:eol-style = native



Mime
View raw message