lucene-solr-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From k...@apache.org
Subject svn commit: r792370 - in /lucene/solr/trunk: ./ src/java/org/apache/solr/analysis/ src/test/org/apache/solr/analysis/ src/webapp/web/admin/
Date Thu, 09 Jul 2009 01:35:30 GMT
Author: koji
Date: Thu Jul  9 01:35:30 2009
New Revision: 792370

URL: http://svn.apache.org/viewvc?rev=792370&view=rev
Log:
SOLR-1256: Show the output of CharFilters in analysis.jsp

Modified:
    lucene/solr/trunk/CHANGES.txt
    lucene/solr/trunk/src/java/org/apache/solr/analysis/CharFilter.java
    lucene/solr/trunk/src/java/org/apache/solr/analysis/CharReader.java
    lucene/solr/trunk/src/java/org/apache/solr/analysis/MappingCharFilter.java
    lucene/solr/trunk/src/test/org/apache/solr/analysis/TestMappingCharFilter.java
    lucene/solr/trunk/src/webapp/web/admin/analysis.jsp

Modified: lucene/solr/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/CHANGES.txt?rev=792370&r1=792369&r2=792370&view=diff
==============================================================================
--- lucene/solr/trunk/CHANGES.txt (original)
+++ lucene/solr/trunk/CHANGES.txt Thu Jul  9 01:35:30 2009
@@ -247,6 +247,9 @@
     This is an advanced debug log file that can be used to aid developers in fixing IndexWriter
bugs. See the commented
     out example in the example solrconfig.xml under the indexDefaults section. 
 	(Chris Harris, Mark Miller)
+
+64. SOLR-1256: Show the output of CharFilters in analysis.jsp. (koji)
+
 Optimizations
 ----------------------
  1. SOLR-374: Use IndexReader.reopen to save resources by re-using parts of the

Modified: lucene/solr/trunk/src/java/org/apache/solr/analysis/CharFilter.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/analysis/CharFilter.java?rev=792370&r1=792369&r2=792370&view=diff
==============================================================================
--- lucene/solr/trunk/src/java/org/apache/solr/analysis/CharFilter.java (original)
+++ lucene/solr/trunk/src/java/org/apache/solr/analysis/CharFilter.java Thu Jul  9 01:35:30
2009
@@ -60,4 +60,16 @@
   public int read(char[] cbuf, int off, int len) throws IOException {
     return input.read(cbuf, off, len);
   }
+
+  public boolean markSupported(){
+    return input.markSupported();
+  }
+
+  public void mark( int readAheadLimit ) throws IOException {
+    input.mark(readAheadLimit);
+  }
+
+  public void reset() throws IOException {
+    input.reset();
+  }
 }

Modified: lucene/solr/trunk/src/java/org/apache/solr/analysis/CharReader.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/analysis/CharReader.java?rev=792370&r1=792369&r2=792370&view=diff
==============================================================================
--- lucene/solr/trunk/src/java/org/apache/solr/analysis/CharReader.java (original)
+++ lucene/solr/trunk/src/java/org/apache/solr/analysis/CharReader.java Thu Jul  9 01:35:30
2009
@@ -54,4 +54,16 @@
   public int read(char[] cbuf, int off, int len) throws IOException {
     return input.read(cbuf, off, len );
   }
+
+  public boolean markSupported(){
+    return input.markSupported();
+  }
+
+  public void mark( int readAheadLimit ) throws IOException {
+    input.mark(readAheadLimit);
+  }
+
+  public void reset() throws IOException {
+    input.reset();
+  }
 }

Modified: lucene/solr/trunk/src/java/org/apache/solr/analysis/MappingCharFilter.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/analysis/MappingCharFilter.java?rev=792370&r1=792369&r2=792370&view=diff
==============================================================================
--- lucene/solr/trunk/src/java/org/apache/solr/analysis/MappingCharFilter.java (original)
+++ lucene/solr/trunk/src/java/org/apache/solr/analysis/MappingCharFilter.java Thu Jul  9
01:35:30 2009
@@ -120,16 +120,4 @@
     }
     return l == 0 ? -1 : l;
   }
-
-  public boolean markSupported(){
-    return false;
-  }
-
-  public void mark( int readAheadLimit ) throws IOException {
-    throw new IOException( "mark/reset not supported" );
-  }
-
-  public void reset() throws IOException {
-    throw new IOException( "mark/reset not supported" );
-  }
 }

Modified: lucene/solr/trunk/src/test/org/apache/solr/analysis/TestMappingCharFilter.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/src/test/org/apache/solr/analysis/TestMappingCharFilter.java?rev=792370&r1=792369&r2=792370&view=diff
==============================================================================
--- lucene/solr/trunk/src/test/org/apache/solr/analysis/TestMappingCharFilter.java (original)
+++ lucene/solr/trunk/src/test/org/apache/solr/analysis/TestMappingCharFilter.java Thu Jul
 9 01:35:30 2009
@@ -42,6 +42,22 @@
     normMap.add( "empty", "" );
   }
 
+  public void testReaderReset() throws Exception {
+    CharStream cs = new MappingCharFilter( normMap, CharReader.get( new StringReader( "x"
) ) );
+    char[] buf = new char[10];
+    int len = cs.read(buf, 0, 10);
+    assertEquals( 1, len );
+    assertEquals( 'x', buf[0]) ;
+    len = cs.read(buf, 0, 10);
+    assertEquals( -1, len );
+
+    // rewind
+    cs.reset();
+    len = cs.read(buf, 0, 10);
+    assertEquals( 1, len );
+    assertEquals( 'x', buf[0]) ;
+  }
+
   public void testNothingChange() throws Exception {
     CharStream cs = new MappingCharFilter( normMap, CharReader.get( new StringReader( "x"
) ) );
     TokenStream ts = new CharStreamAwareWhitespaceTokenizer( cs );

Modified: lucene/solr/trunk/src/webapp/web/admin/analysis.jsp
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/src/webapp/web/admin/analysis.jsp?rev=792370&r1=792369&r2=792370&view=diff
==============================================================================
--- lucene/solr/trunk/src/webapp/web/admin/analysis.jsp (original)
+++ lucene/solr/trunk/src/webapp/web/admin/analysis.jsp Thu Jul  9 01:35:30 2009
@@ -19,6 +19,9 @@
                  org.apache.lucene.analysis.Token,
                  org.apache.lucene.analysis.TokenStream,
                  org.apache.lucene.index.Payload,
+                 org.apache.solr.analysis.CharReader,
+                 org.apache.solr.analysis.CharStream,
+                 org.apache.solr.analysis.CharFilterFactory,
                  org.apache.solr.analysis.TokenFilterFactory,
                  org.apache.solr.analysis.TokenizerChain,
                  org.apache.solr.analysis.TokenizerFactory,
@@ -171,19 +174,32 @@
 
 <%!
   private static void doAnalyzer(JspWriter out, SchemaField field, String val, boolean queryAnalyser,
boolean verbose, Set<Tok> match) throws Exception {
-    Reader reader = new StringReader(val);
+    CharStream reader = CharReader.get(new StringReader(val));
 
     FieldType ft = field.getType();
      Analyzer analyzer = queryAnalyser ?
              ft.getQueryAnalyzer() : ft.getAnalyzer();
      if (analyzer instanceof TokenizerChain) {
        TokenizerChain tchain = (TokenizerChain)analyzer;
+       CharFilterFactory[] cfiltfacs = tchain.getCharFilterFactories();
        TokenizerFactory tfac = tchain.getTokenizerFactory();
        TokenFilterFactory[] filtfacs = tchain.getTokenFilterFactories();
 
-       TokenStream tstream = tfac.create(tchain.charStream(reader));
+       if( cfiltfacs != null ){
+         for(CharFilterFactory cfiltfac : cfiltfacs ){
+           reader = cfiltfac.create(reader);
+           if(verbose){
+             writeHeader(out, cfiltfac.getClass(), cfiltfac.getArgs());
+             writeCharStream(out, reader);
+           }
+         }
+       }
+
+       // StringReader should support reset()
+       reader.reset();
+       TokenStream tstream = tfac.create(reader);
        List<Token> tokens = getTokens(tstream);
-       tstream = tfac.create(tchain.charStream(reader));
+       tstream = tfac.create(reader);
        if (verbose) {
          writeHeader(out, tfac.getClass(), tfac.getArgs());
        }
@@ -453,4 +469,30 @@
     out.println("</table>");
   }
 
+  static void writeCharStream(JspWriter out, CharStream input) throws IOException {
+    out.println("<table width=\"auto\" class=\"analysis\" border=\"1\">");
+    out.println("<tr>");
+
+    out.print("<th NOWRAP>");
+    XML.escapeCharData("text",out);
+    out.println("</th>");
+
+    // StringReader should support reset()
+    input.reset();
+    final int BUFFER_SIZE = 1024;
+    char[] buf = new char[BUFFER_SIZE];
+    int len = 0;
+    StringBuilder sb = new StringBuilder();
+    do {
+      len = input.read( buf, 0, BUFFER_SIZE );
+      sb.append(buf, 0, len);
+    } while( len == BUFFER_SIZE );
+    out.print("<td class=\"debugdata\">");
+    XML.escapeCharData(sb.toString(),out);
+    out.println("</td>");
+    
+    out.println("</tr>");
+    out.println("</table>");
+  }
+
 %>



Mime
View raw message