lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sar...@apache.org
Subject svn commit: r1725869 - in /lucene/dev/trunk: lucene/ lucene/core/src/java/org/apache/lucene/util/ lucene/core/src/test/org/apache/lucene/util/ solr/ solr/core/src/java/org/apache/solr/schema/ solr/core/src/test-files/solr/collection1/conf/ solr/core/sr...
Date Thu, 21 Jan 2016 05:51:36 GMT
Author: sarowe
Date: Thu Jan 21 05:51:36 2016
New Revision: 1725869

URL: http://svn.apache.org/viewvc?rev=1725869&view=rev
Log:
SOLR-4619: Improve PreAnalyzedField query analysis

Modified:
    lucene/dev/trunk/lucene/CHANGES.txt
    lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/AttributeSource.java
    lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/util/TestAttributeSource.java
    lucene/dev/trunk/solr/CHANGES.txt
    lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/PreAnalyzedField.java
    lucene/dev/trunk/solr/core/src/test-files/solr/collection1/conf/schema-preanalyzed.xml
    lucene/dev/trunk/solr/core/src/test/org/apache/solr/schema/PreAnalyzedFieldTest.java
    lucene/dev/trunk/solr/server/solr/configsets/sample_techproducts_configs/conf/managed-schema

Modified: lucene/dev/trunk/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/CHANGES.txt?rev=1725869&r1=1725868&r2=1725869&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/CHANGES.txt (original)
+++ lucene/dev/trunk/lucene/CHANGES.txt Thu Jan 21 05:51:36 2016
@@ -126,6 +126,9 @@ New Features
 * LUCENE-6818: Add DFISimilarity implementing the divergence from independence
   model. (Ahmet Arslan via Robert Muir)
 
+* SOLR-4619: Added removeAllAttributes() to AttributeSource, which removes
+  all previously added attributes.
+
 API Changes
 
 * LUCENE-6908: GeoUtils static relational methods have been refactored to new 

Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/AttributeSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/AttributeSource.java?rev=1725869&r1=1725868&r2=1725869&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/AttributeSource.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/AttributeSource.java Thu
Jan 21 05:51:36 2016
@@ -270,7 +270,15 @@ public class AttributeSource {
       state.attribute.clear();
     }
   }
-  
+
+  /**
+   * Removes all attributes and their implementations from this AttributeSource.
+   */
+  public final void removeAllAttributes() {
+    attributes.clear();
+    attributeImpls.clear();
+  }
+
   /**
    * Captures the state of all Attributes. The return value can be passed to
    * {@link #restoreState} to restore the state of this or another AttributeSource.

Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/util/TestAttributeSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/util/TestAttributeSource.java?rev=1725869&r1=1725868&r2=1725869&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/util/TestAttributeSource.java
(original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/util/TestAttributeSource.java
Thu Jan 21 05:51:36 2016
@@ -18,9 +18,12 @@ package org.apache.lucene.util;
  */
 
 import org.apache.lucene.analysis.Token;
+import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.tokenattributes.*;
 
+import java.util.ArrayList;
 import java.util.Iterator;
+import java.util.List;
 
 public class TestAttributeSource extends LuceneTestCase {
 
@@ -175,5 +178,52 @@ public class TestAttributeSource extends
     clone.getPayload().bytes[0] = 10; // modify one byte, srcBytes shouldn't change
     assertEquals("clone() wasn't deep", 1, src.getPayload().bytes[0]);
   }
-  
+
+  public void testRemoveAllAttributes() {
+    List<Class<? extends Attribute>> attrClasses = new ArrayList<>();
+    attrClasses.add(CharTermAttribute.class);
+    attrClasses.add(OffsetAttribute.class);
+    attrClasses.add(FlagsAttribute.class);
+    attrClasses.add(PayloadAttribute.class);
+    attrClasses.add(PositionIncrementAttribute.class);
+    attrClasses.add(TypeAttribute.class);
+
+    // Add attributes with the default factory, then try to remove all of them
+    AttributeSource defaultFactoryAttributeSource = new AttributeSource();
+
+    assertFalse(defaultFactoryAttributeSource.hasAttributes());
+
+    for (Class<? extends Attribute> attrClass : attrClasses) {
+      defaultFactoryAttributeSource.addAttribute(attrClass);
+      assertTrue("Missing added attribute " + attrClass.getSimpleName(),
+          defaultFactoryAttributeSource.hasAttribute(attrClass));
+    }
+
+    defaultFactoryAttributeSource.removeAllAttributes();
+
+    for (Class<? extends Attribute> attrClass : attrClasses) {
+      assertFalse("Didn't remove attribute " + attrClass.getSimpleName(),
+          defaultFactoryAttributeSource.hasAttribute(attrClass));
+    }
+    assertFalse(defaultFactoryAttributeSource.hasAttributes());
+
+    // Add attributes with the packed implementations factory, then try to remove all of
them
+    AttributeSource packedImplsAttributeSource
+        = new AttributeSource(TokenStream.DEFAULT_TOKEN_ATTRIBUTE_FACTORY);
+    assertFalse(packedImplsAttributeSource.hasAttributes());
+
+    for (Class<? extends Attribute> attrClass : attrClasses) {
+      packedImplsAttributeSource.addAttribute(attrClass);
+      assertTrue("Missing added attribute " + attrClass.getSimpleName(),
+          packedImplsAttributeSource.hasAttribute(attrClass));
+    }
+
+    packedImplsAttributeSource.removeAllAttributes();
+
+    for (Class<? extends Attribute> attrClass : attrClasses) {
+      assertFalse("Didn't remove attribute " + attrClass.getSimpleName(),
+          packedImplsAttributeSource.hasAttribute(attrClass));
+    }
+    assertFalse(packedImplsAttributeSource.hasAttributes());
+  }
 }

Modified: lucene/dev/trunk/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/CHANGES.txt?rev=1725869&r1=1725868&r2=1725869&view=diff
==============================================================================
--- lucene/dev/trunk/solr/CHANGES.txt (original)
+++ lucene/dev/trunk/solr/CHANGES.txt Thu Jan 21 05:51:36 2016
@@ -331,6 +331,8 @@ New Features
   create/delete shard, delete replica, add/delete replica property, add/remove role,
   overseer status, balance shard unique, rebalance leaders, modify collection, migrate state
format (Varun Thacker)
 
+* SOLR-4619: Improve PreAnalyzedField query analysis. (Andrzej Bialecki, Steve Rowe)
+
 Bug Fixes
 ----------------------
 

Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/PreAnalyzedField.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/PreAnalyzedField.java?rev=1725869&r1=1725868&r2=1725869&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/PreAnalyzedField.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/PreAnalyzedField.java Thu Jan
21 05:51:36 2016
@@ -51,7 +51,7 @@ import static org.apache.solr.common.par
  * Pre-analyzed field type provides a way to index a serialized token stream,
  * optionally with an independent stored value of a field.
  */
-public class PreAnalyzedField extends FieldType {
+public class PreAnalyzedField extends TextField {
   private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
 
   /** Init argument name. Value is a fully-qualified class name of the parser
@@ -63,8 +63,8 @@ public class PreAnalyzedField extends Fi
 
   
   private PreAnalyzedParser parser;
-  private Analyzer analyzer;
-  
+  private PreAnalyzedAnalyzer preAnalyzer;
+
   @Override
   public void init(IndexSchema schema, Map<String, String> args) {
     super.init(schema, args);
@@ -91,22 +91,31 @@ public class PreAnalyzedField extends Fi
       args.remove(PARSER_IMPL);
     }
     // create Analyzer instance for reuse:
-    analyzer = new SolrAnalyzer() {
-      @Override
-      protected TokenStreamComponents createComponents(String fieldName) {
-        return new TokenStreamComponents(new PreAnalyzedTokenizer(parser));
-      }
-    };
+    preAnalyzer = new PreAnalyzedAnalyzer(parser);
   }
 
+  /**
+   * Overridden to return an analyzer consisting of a {@link PreAnalyzedTokenizer}.
+   * NOTE: If an index analyzer is specified in the schema, it will be ignored.
+   */
   @Override
   public Analyzer getIndexAnalyzer() {
-    return analyzer;
+    return preAnalyzer;
   }
-  
+
+  /**
+   * Returns the query analyzer defined via the schema, unless there is none,
+   * in which case the index-time pre-analyzer is returned.
+   *
+   * Note that if the schema specifies an index-time analyzer via either
+   * {@code <analyzer>} or {@code <analyzer type="index">}, but no query-time
+   * analyzer, the query analyzer returned here will be the index-time
+   * analyzer specified in the schema rather than the pre-analyzer.
+   */
   @Override
   public Analyzer getQueryAnalyzer() {
-    return getIndexAnalyzer();
+    Analyzer queryAnalyzer = super.getQueryAnalyzer();
+    return queryAnalyzer instanceof FieldType.DefaultAnalyzer ? getIndexAnalyzer() : queryAnalyzer;
   }
 
   @Override
@@ -221,8 +230,10 @@ public class PreAnalyzedField extends Fi
       return null;
     }
     PreAnalyzedTokenizer parse = new PreAnalyzedTokenizer(parser);
-    parse.setReader(new StringReader(val));
-    parse.reset(); // consume
+    Reader reader = new StringReader(val);
+    parse.setReader(reader);
+    parse.decodeInput(reader); // consume
+    parse.reset();
     org.apache.lucene.document.FieldType type = createFieldType(field);
     if (type == null) {
       parse.close();
@@ -263,7 +274,7 @@ public class PreAnalyzedField extends Fi
     }
     return f;
   }
-    
+
   /**
    * Token stream that works from a list of saved states.
    */
@@ -273,7 +284,8 @@ public class PreAnalyzedField extends Fi
     private String stringValue = null;
     private byte[] binaryValue = null;
     private PreAnalyzedParser parser;
-    
+    private IOException readerConsumptionException;
+
     public PreAnalyzedTokenizer(PreAnalyzedParser parser) {
       // we don't pack attributes: since we are used for (de)serialization and dont want
bloat.
       super(AttributeFactory.DEFAULT_ATTRIBUTE_FACTORY);
@@ -294,29 +306,44 @@ public class PreAnalyzedField extends Fi
 
     @Override
     public final boolean incrementToken() {
-      // lazy init the iterator
-      if (it == null) {
-        it = cachedStates.iterator();
-      }
-    
       if (!it.hasNext()) {
         return false;
       }
       
-      AttributeSource.State state = (State) it.next();
+      AttributeSource.State state = it.next();
       restoreState(state.clone());
       return true;
     }
-  
+
+    /**
+     * Throws a delayed exception if one was thrown from decodeInput()
+     * while reading from the input reader.
+     */
     @Override
     public final void reset() throws IOException {
-      // NOTE: this acts like rewind if you call it again
-      if (it == null) {
-        super.reset();
-        cachedStates.clear();
-        stringValue = null;
-        binaryValue = null;
-        ParseResult res = parser.parse(input, this);
+      super.reset();
+      if (readerConsumptionException != null) {
+        IOException e = new IOException(readerConsumptionException);
+        readerConsumptionException = null;
+        throw e;
+      }
+      it = cachedStates.iterator();
+    }
+
+    private void setReaderConsumptionException(IOException e) {
+      readerConsumptionException = e;
+    }
+
+    /**
+     * Parses the input reader and adds attributes specified there.
+     */
+    private void decodeInput(Reader reader) throws IOException {
+      removeAllAttributes();  // reset attributes to the empty set
+      cachedStates.clear();
+      stringValue = null;
+      binaryValue = null;
+      try {
+        ParseResult res = parser.parse(reader, this);
         if (res != null) {
           stringValue = res.str;
           binaryValue = res.bin;
@@ -324,9 +351,35 @@ public class PreAnalyzedField extends Fi
             cachedStates.addAll(res.states);
           }
         }
+      } catch (IOException e) {
+        removeAllAttributes();  // reset attributes to the empty set
+        throw e; // rethrow
       }
-      it = cachedStates.iterator();
     }
   }
-  
+
+  private static class PreAnalyzedAnalyzer extends SolrAnalyzer {
+    private PreAnalyzedParser parser;
+
+    PreAnalyzedAnalyzer(PreAnalyzedParser parser) {
+      this.parser = parser;
+    }
+
+    @Override
+    protected TokenStreamComponents createComponents(String fieldName) {
+      final PreAnalyzedTokenizer tokenizer = new PreAnalyzedTokenizer(parser);
+      return new TokenStreamComponents(tokenizer) {
+        @Override
+        protected void setReader(final Reader reader) {
+          super.setReader(reader);
+          try {
+            tokenizer.decodeInput(reader);
+          } catch (IOException e) {
+            // save this exception for reporting when reset() is called
+            tokenizer.setReaderConsumptionException(e);
+          }
+        }
+      };
+    }
+  }
 }

Modified: lucene/dev/trunk/solr/core/src/test-files/solr/collection1/conf/schema-preanalyzed.xml
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test-files/solr/collection1/conf/schema-preanalyzed.xml?rev=1725869&r1=1725868&r2=1725869&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/test-files/solr/collection1/conf/schema-preanalyzed.xml
(original)
+++ lucene/dev/trunk/solr/core/src/test-files/solr/collection1/conf/schema-preanalyzed.xml
Thu Jan 21 05:51:36 2016
@@ -18,7 +18,18 @@
 <schema name="tiny" version="1.1">
 
   <types>
-    <fieldType name="preanalyzed" class="solr.PreAnalyzedField" parserImpl="json"/>
+    <fieldType name="preanalyzed-no-analyzer" class="solr.PreAnalyzedField" parserImpl="json"/>
+    <fieldType name="preanalyzed-with-analyzer" class="solr.PreAnalyzedField">
+      <analyzer>
+        <tokenizer class="solr.WhitespaceTokenizerFactory"/>
+      </analyzer>
+    </fieldType>
+    <fieldType name="preanalyzed-with-query-analyzer" class="solr.PreAnalyzedField">
+      <analyzer type="query">
+        <tokenizer class="solr.StandardTokenizerFactory"/>
+        <filter class="solr.LowerCaseFilterFactory"/>
+      </analyzer>
+    </fieldType>
     <fieldType name="string" class="solr.StrField"/>
     <fieldType name="long" class="solr.TrieLongField" precisionStep="0" omitNorms="true"
positionIncrementGap="0"/>
   </types>
@@ -26,7 +37,9 @@
   <fields>
     <field name="id" type="string" indexed="true" stored="true" required="true"/>
     <field name="_version_" type="long" indexed="true" stored="true" multiValued="false"/>
-    <field name="pre" type="preanalyzed" indexed="true" stored="true" multiValued="false"/>
+    <field name="pre_no_analyzer" type="preanalyzed-no-analyzer" indexed="true" stored="true"
multiValued="false"/>
+    <field name="pre_with_analyzer" type="preanalyzed-with-analyzer" indexed="true" stored="true"
multiValued="false"/>
+    <field name="pre_with_query_analyzer" type="preanalyzed-with-query-analyzer" indexed="true"
stored="true" multiValued="false"/>
   </fields>
 
   <uniqueKey>id</uniqueKey>

Modified: lucene/dev/trunk/solr/core/src/test/org/apache/solr/schema/PreAnalyzedFieldTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test/org/apache/solr/schema/PreAnalyzedFieldTest.java?rev=1725869&r1=1725868&r2=1725869&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/test/org/apache/solr/schema/PreAnalyzedFieldTest.java (original)
+++ lucene/dev/trunk/solr/core/src/test/org/apache/solr/schema/PreAnalyzedFieldTest.java Thu
Jan 21 05:51:36 2016
@@ -20,6 +20,9 @@ package org.apache.solr.schema;
 import java.util.Collections;
 import java.util.HashMap;
 
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.document.Field;
 import org.apache.solr.SolrTestCaseJ4;
 import org.apache.solr.schema.PreAnalyzedField.PreAnalyzedParser;
@@ -52,7 +55,7 @@ public class PreAnalyzedFieldTest extend
     "1 one,p=deadbeef,s=0,e=3 two,p=0123456789abcdef,s=4,e=7 three,s=8,e=13"
   };
 
-  private static final String[] invalid = {
+  private static final String[] invalidSimple = {
     "one two three", // missing version #
     "2 one two three", // invalid version #
     "1 o,ne two", // missing escape
@@ -65,7 +68,18 @@ public class PreAnalyzedFieldTest extend
     "1 =stored ", // unterminated stored
     "1 ===" // empty stored (ok), but unescaped = in token stream
   };
-  
+
+  private static final String validJson
+      = json("{'v':'1','str':'stored-value','tokens':[{'t':'a'},{'t':'b'},{'t':'c'}]}");
+
+  private static final String[] invalidJson = {
+      json("'v':'1','str':'stored-value','tokens':[{'t':'a'},{'t':'b'},{'t':'c'}]"),    //
missing enclosing object
+      json("{'str':'stored-value','tokens':[{'t':'a'},{'t':'b'},{'t':'c'}]}"),          //
missing version #
+      json("{'v':'2','str':'stored-value','tokens':[{'t':'a'},{'t':'b'},{'t':'c'}]}"),  //
invalid version #
+      json("{'v':'1','str':'stored-value','tokens':[{}]}"),                             //
single token no attribs
+      json("{'v':'1','str':'stored-value','tokens':[{'t'}]}"),                          //
missing attrib value
+  };
+
   SchemaField field = null;
   int props = 
     FieldProperties.INDEXED | FieldProperties.STORED;
@@ -102,17 +116,69 @@ public class PreAnalyzedFieldTest extend
     }
   }
 
+  private String addTwoDocs(int firstId, String field) {
+    return "<add>\n"
+        + doc("id", Integer.toString(firstId), field,
+              json("{'v':'1','str':'document one','tokens':[{'t':'one'},{'t':'two'},{'t':'three','i':100}]}"))
+        + doc("id", Integer.toString(firstId + 1), field,
+              json("{'v':'1','str':'document two','tokens':[{'t':'eleven'},{'t':'twelve'},{'t':'thirteen','i':110}]}"))
+        + "</add>\n";
+  }
+
   @Test
-  public void testValidSimple2() {
-    assertU(adoc("id", "1",
-                 "pre", "{\"v\":\"1\",\"str\":\"document one\",\"tokens\":[{\"t\":\"one\"},{\"t\":\"two\"},{\"t\":\"three\",\"i\":100}]}"));
+  public void testIndexAndQueryNoSchemaAnalyzer() throws Exception {
+    assertU(addTwoDocs(1, "pre_no_analyzer"));
+    assertU(commit());
+    assertQ(req("q", "id:(1 2)", "sort", "id asc")
+        ,"//result[@numFound='2']"
+        ,"//result/doc[1]/str[@name='id'][.='1']"
+        ,"//result/doc[1]/str[@name='pre_no_analyzer'][.='document one']"
+        ,"//result/doc[2]/str[@name='id'][.='2']"
+        ,"//result/doc[2]/str[@name='pre_no_analyzer'][.='document two']"
+    );
+    assertQ(req("q", "{!field f='pre_no_analyzer'}{'v':'1','tokens':[{'t':'two'}]}")
+        ,"//result[@numFound='1']"
+    );
+    assertQ(req("q", "{!field f='pre_no_analyzer'}{'v':'1','tokens':[{'t':'eleven'},{'t':'twelve'}]}")
+        ,"//result[@numFound='1']"
+    );
   }
-  
+
+  @Test
+  public void testIndexAndQueryWithSchemaAnalyzer() {
+    assertU(addTwoDocs(3, "pre_with_analyzer"));
+    assertU(commit());
+    assertQ(req("q", "id:(3 4)", "sort", "id asc")
+        ,"//result[@numFound='2']"
+        ,"//result/doc[1]/str[@name='id'][.='3']"
+        ,"//result/doc[1]/str[@name='pre_with_analyzer'][.='document one']"
+        ,"//result/doc[2]/str[@name='id'][.='4']"
+        ,"//result/doc[2]/str[@name='pre_with_analyzer'][.='document two']"
+    );
+    assertQ(req("q", "pre_with_analyzer:(+two +three)"), "//result[@numFound='1']");
+    assertQ(req("q", "pre_with_analyzer:(+eleven +twelve)"), "//result[@numFound='1']");
+  }
+
+  @Test
+  public void testIndexAndQueryWithSchemaQueryAnalyzer() {
+    assertU(addTwoDocs(5, "pre_with_query_analyzer"));
+    assertU(commit());
+    assertQ(req("q", "id:(5 6)", "sort", "id asc")
+        ,"//result[@numFound='2']"
+        ,"//result/doc[1]/str[@name='id'][.='5']"
+        ,"//result/doc[1]/str[@name='pre_with_query_analyzer'][.='document one']"
+        ,"//result/doc[2]/str[@name='id'][.='6']"
+        ,"//result/doc[2]/str[@name='pre_with_query_analyzer'][.='document two']"
+    );
+    assertQ(req("q", "pre_with_query_analyzer:one,two"), "//result[@numFound='1']");
+    assertQ(req("q", "pre_with_query_analyzer:eleven,twelve"), "//result[@numFound='1']");
+  }
+
   @Test
   public void testInvalidSimple() {
     PreAnalyzedField paf = new PreAnalyzedField();
     paf.init(h.getCore().getLatestSchema(), Collections.<String,String>emptyMap());
-    for (String s : invalid) {
+    for (String s : invalidSimple) {
       try {
         paf.fromString(field, s, 1.0f);
         fail("should fail: '" + s + "'");
@@ -121,6 +187,35 @@ public class PreAnalyzedFieldTest extend
       }
     }
   }
+
+  public void testInvalidJson() throws Exception {
+    PreAnalyzedField paf = new PreAnalyzedField();
+    paf.init(h.getCore().getLatestSchema(), Collections.emptyMap());
+    Analyzer preAnalyzer = paf.getIndexAnalyzer();
+    for (String s: invalidJson) {
+      TokenStream stream = null;
+      try {
+        stream = preAnalyzer.tokenStream("dummy", s);
+        stream.reset(); // exception should be triggered here.
+        fail("should fail: '" + s + "'");
+      } catch (Exception e) {
+        // expected
+      } finally {
+        if (stream != null) {
+          stream.close();
+        }
+      }
+    }
+    // make sure the analyzer can now handle properly formatted input
+    TokenStream stream = preAnalyzer.tokenStream("dummy", validJson);
+    CharTermAttribute termAttr = stream.addAttribute(CharTermAttribute.class);
+    stream.reset();
+    while (stream.incrementToken()) {
+      assertFalse("zero-length token", termAttr.length() == 0);
+    }
+    stream.end();
+    stream.close();
+  }
   
   // "1 =test ąćęłńóśźż \u0001=one,i=22,s=123,e=128,p=deadbeef,y=word
two,i=1,s=5,e=8,y=word three,i=1,s=20,e=22,y=foobar"
   
@@ -147,7 +242,7 @@ public class PreAnalyzedFieldTest extend
     paf.init(h.getCore().getLatestSchema(), args);
     try {
       Field f = (Field)paf.fromString(field, valid[0], 1.0f);
-      fail("Should fail JSON parsing: '" + valid[0]);
+      fail("Should fail JSON parsing: '" + valid[0] + "'");
     } catch (Exception e) {
     }
     byte[] deadbeef = new byte[]{(byte)0xd, (byte)0xe, (byte)0xa, (byte)0xd, (byte)0xb, (byte)0xe,
(byte)0xe, (byte)0xf};

Modified: lucene/dev/trunk/solr/server/solr/configsets/sample_techproducts_configs/conf/managed-schema
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/server/solr/configsets/sample_techproducts_configs/conf/managed-schema?rev=1725869&r1=1725868&r2=1725869&view=diff
==============================================================================
--- lucene/dev/trunk/solr/server/solr/configsets/sample_techproducts_configs/conf/managed-schema
(original)
+++ lucene/dev/trunk/solr/server/solr/configsets/sample_techproducts_configs/conf/managed-schema
Thu Jan 21 05:51:36 2016
@@ -129,7 +129,8 @@
      tokens
    -->   
    <field name="id" type="string" indexed="true" stored="true" required="true" multiValued="false"
/> 
-        
+
+   <field name="pre" type="preanalyzed" indexed="true" stored="true"/>
    <field name="sku" type="text_en_splitting_tight" indexed="true" stored="true" omitNorms="true"/>
    <field name="name" type="text_general" indexed="true" stored="true"/>
    <field name="manu" type="text_general" indexed="true" stored="true" omitNorms="true"/>
@@ -1138,7 +1139,15 @@
         <filter class="solr.SnowballPorterFilterFactory" language="Turkish"/>
       </analyzer>
     </fieldType>
-  
+
+    <!-- Pre-analyzed field type, allows inserting arbitrary token streams and stored
values. -->
+    <fieldType name="preanalyzed" class="solr.PreAnalyzedField">
+      <!-- PreAnalyzedField's builtin index analyzer just decodes the pre-analyzed token
stream. -->
+      <analyzer type="query">
+        <tokenizer class="solr.WhitespaceTokenizerFactory"/>
+      </analyzer>
+    </fieldType>
+
   <!-- Similarity is the scoring routine for each document vs. a query.
        A custom Similarity or SimilarityFactory may be specified here, but 
        the default is fine for most applications.  



Mime
View raw message