lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hoss...@apache.org
Subject svn commit: r944463 - in /lucene/dev/trunk/solr: ./ src/java/org/apache/solr/update/processor/ src/test/org/apache/solr/update/processor/ src/test/test-files/solr/conf/
Date Fri, 14 May 2010 20:58:30 GMT
Author: hossman
Date: Fri May 14 20:58:29 2010
New Revision: 944463

URL: http://svn.apache.org/viewvc?rev=944463&view=rev
Log:
SOLR-1908: Fixed SignatureUpdateProcessor to fail to initialize on invalid config. Specificly:
a signatureField that does not exist, or overwriteDupes=true with a signatureField that is
not indexed.

Modified:
    lucene/dev/trunk/solr/CHANGES.txt
    lucene/dev/trunk/solr/src/java/org/apache/solr/update/processor/SignatureUpdateProcessorFactory.java
    lucene/dev/trunk/solr/src/test/org/apache/solr/update/processor/SignatureUpdateProcessorFactoryTest.java
    lucene/dev/trunk/solr/src/test/test-files/solr/conf/bad-schema.xml
    lucene/dev/trunk/solr/src/test/test-files/solr/conf/schema-copyfield-test.xml
    lucene/dev/trunk/solr/src/test/test-files/solr/conf/schema-luceneMatchVersion.xml
    lucene/dev/trunk/solr/src/test/test-files/solr/conf/schema-not-required-unique-key.xml
    lucene/dev/trunk/solr/src/test/test-files/solr/conf/schema-required-fields.xml
    lucene/dev/trunk/solr/src/test/test-files/solr/conf/schema-reversed.xml
    lucene/dev/trunk/solr/src/test/test-files/solr/conf/schema-trie.xml
    lucene/dev/trunk/solr/src/test/test-files/solr/conf/schema11.xml
    lucene/dev/trunk/solr/src/test/test-files/solr/conf/solrconfig.xml

Modified: lucene/dev/trunk/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/CHANGES.txt?rev=944463&r1=944462&r2=944463&view=diff
==============================================================================
--- lucene/dev/trunk/solr/CHANGES.txt (original)
+++ lucene/dev/trunk/solr/CHANGES.txt Fri May 14 20:58:29 2010
@@ -287,6 +287,11 @@ Bug Fixes
 
 * SOLR-1902: Exposed SolrResourceLoader's class loader for use by Tika  
 
+* SOLR-1908: Fixed SignatureUpdateProcessor to fail to initialize on
+  invalid config.  Specificly: a signatureField that does not exist,
+  or overwriteDupes=true with a signatureField that is not indexed.
+  (hossman)
+
 Other Changes
 ----------------------
 

Modified: lucene/dev/trunk/solr/src/java/org/apache/solr/update/processor/SignatureUpdateProcessorFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/src/java/org/apache/solr/update/processor/SignatureUpdateProcessorFactory.java?rev=944463&r1=944462&r2=944463&view=diff
==============================================================================
--- lucene/dev/trunk/solr/src/java/org/apache/solr/update/processor/SignatureUpdateProcessorFactory.java
(original)
+++ lucene/dev/trunk/solr/src/java/org/apache/solr/update/processor/SignatureUpdateProcessorFactory.java
Fri May 14 20:58:29 2010
@@ -23,6 +23,8 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.lucene.index.Term;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrException.ErrorCode;
 import org.apache.solr.common.SolrInputDocument;
 import org.apache.solr.common.SolrInputField;
 import org.apache.solr.common.params.SolrParams;
@@ -34,9 +36,14 @@ import org.apache.solr.update.AddUpdateC
 import org.apache.solr.update.CommitUpdateCommand;
 import org.apache.solr.update.DeleteUpdateCommand;
 import org.apache.solr.core.SolrResourceLoader;
-
-public class SignatureUpdateProcessorFactory extends
-    UpdateRequestProcessorFactory {
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.schema.IndexSchema;
+import org.apache.solr.schema.SchemaField;
+import org.apache.solr.util.plugin.SolrCoreAware;
+
+public class SignatureUpdateProcessorFactory 
+  extends UpdateRequestProcessorFactory 
+  implements SolrCoreAware {
 
   private List<String> sigFields;
   private String signatureField;
@@ -72,6 +79,23 @@ public class SignatureUpdateProcessorFac
     }
   }
 
+  public void inform(SolrCore core) {
+    final SchemaField field = core.getSchema().getFieldOrNull(getSignatureField());
+    if (null == field) {
+      throw new SolrException
+        (ErrorCode.SERVER_ERROR,
+         "Can't use signatureField which does not exist in schema: "
+         + getSignatureField());
+    }
+
+    if (getOverwriteDupes() && ( ! field.indexed() ) ) {
+      throw new SolrException
+        (ErrorCode.SERVER_ERROR,
+         "Can't set overwriteDupes when signatureField is not indexed: "
+         + getSignatureField());
+    }
+  }
+
   public List<String> getSigFields() {
     return sigFields;
   }

Modified: lucene/dev/trunk/solr/src/test/org/apache/solr/update/processor/SignatureUpdateProcessorFactoryTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/src/test/org/apache/solr/update/processor/SignatureUpdateProcessorFactoryTest.java?rev=944463&r1=944462&r2=944463&view=diff
==============================================================================
--- lucene/dev/trunk/solr/src/test/org/apache/solr/update/processor/SignatureUpdateProcessorFactoryTest.java
(original)
+++ lucene/dev/trunk/solr/src/test/org/apache/solr/update/processor/SignatureUpdateProcessorFactoryTest.java
Fri May 14 20:58:29 2010
@@ -24,6 +24,7 @@ import java.util.Map;
 import org.apache.solr.common.params.MultiMapSolrParams;
 import org.apache.solr.common.params.SolrParams;
 import org.apache.solr.common.params.UpdateParams;
+import org.apache.solr.common.util.NamedList;
 import org.apache.solr.common.util.ContentStream;
 import org.apache.solr.common.util.ContentStreamBase;
 import org.apache.solr.core.SolrCore;
@@ -37,6 +38,9 @@ import org.apache.solr.util.AbstractSolr
  */
 public class SignatureUpdateProcessorFactoryTest extends AbstractSolrTestCase {
 
+  /** modified by tests as needed */
+  private String processor = "dedupe";
+
   @Override
   public String getSchemaFile() {
     return "schema12.xml";
@@ -47,6 +51,12 @@ public class SignatureUpdateProcessorFac
     return "solrconfig.xml";
   }
 
+  @Override
+  public void setUp() throws Exception {
+    super.setUp();
+    processor = "dedupe"; // set the default that most tests expect
+  }
+
   public void testDupeDetection() throws Exception {
     SolrCore core = h.getCore();
     UpdateRequestProcessorChain chained = core.getUpdateProcessingChain(
@@ -169,10 +179,45 @@ public class SignatureUpdateProcessorFac
     factory.setEnabled(false);
   }
 
+  /**
+   * a non-indexed signatureField is fine as long as overwriteDupes==false
+   */
+  public void testNonIndexedSignatureField() throws Exception {
+    SolrCore core = h.getCore();
+
+    assertEquals("docs found when none are expected at start",
+                 0l, core.getSearcher().get().getReader().numDocs());
+
+    processor = "stored_sig";
+    addDoc(adoc("id", "2a", "v_t", "Hello Dude man!", "name", "ali babi'"));
+    addDoc(adoc("id", "2b", "v_t", "Hello Dude man!", "name", "ali babi'"));
+    addDoc(commit());
+
+    assertEquals("did not find exepcted docs",
+                 2l, core.getSearcher().get().getReader().numDocs());
+  }
+
+  public void testFailNonIndexedSigWithOverwriteDupes() throws Exception {
+    SolrCore core = h.getCore();
+    SignatureUpdateProcessorFactory f = new SignatureUpdateProcessorFactory();
+    NamedList<String> initArgs = new NamedList<String>();
+    initArgs.add("overwriteDupes", "true");
+    initArgs.add("signatureField", "signatureField_sS");
+    f.init(initArgs);
+    boolean exception_ok = false;
+    try {
+      f.inform(core);
+    } catch (Exception e) {
+      exception_ok = true;
+    }
+    assertTrue("Should have gotten an exception from inform(SolrCore)", 
+               exception_ok);
+  }
+
   private void addDoc(String doc) throws Exception {
     Map<String, String[]> params = new HashMap<String, String[]>();
     MultiMapSolrParams mmparams = new MultiMapSolrParams(params);
-    params.put(UpdateParams.UPDATE_PROCESSOR, new String[] { "dedupe" });
+    params.put(UpdateParams.UPDATE_PROCESSOR, new String[] { processor });
     SolrQueryRequestBase req = new SolrQueryRequestBase(h.getCore(),
         (SolrParams) mmparams) {
     };

Modified: lucene/dev/trunk/solr/src/test/test-files/solr/conf/bad-schema.xml
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/src/test/test-files/solr/conf/bad-schema.xml?rev=944463&r1=944462&r2=944463&view=diff
==============================================================================
--- lucene/dev/trunk/solr/src/test/test-files/solr/conf/bad-schema.xml (original)
+++ lucene/dev/trunk/solr/src/test/test-files/solr/conf/bad-schema.xml Fri May 14 20:58:29
2010
@@ -34,6 +34,8 @@
    <field name="id" type="text" indexed="true" stored="true" multiValued="false" required="false"/>
    
    <fieldType name="string" class="solr.StrField"/>
+
+   <field name="signatureField" type="string" indexed="true" stored="false"/>
     
    <field name="fAgain" type="text" indexed="true" stored="true"/>
    <field name="fAgain" type="text" indexed="true" stored="true"/>
@@ -41,6 +43,8 @@
    <dynamicField name="*_twice"  type="text"  indexed="true"  stored="true"/>
    <dynamicField name="*_twice"  type="text"  indexed="true"  stored="true"/>
 
+   <dynamicField name="*_sS" type="string"  indexed="false" stored="true"/>
+
  </fields>
 
  <defaultSearchField>id</defaultSearchField>

Modified: lucene/dev/trunk/solr/src/test/test-files/solr/conf/schema-copyfield-test.xml
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/src/test/test-files/solr/conf/schema-copyfield-test.xml?rev=944463&r1=944462&r2=944463&view=diff
==============================================================================
--- lucene/dev/trunk/solr/src/test/test-files/solr/conf/schema-copyfield-test.xml (original)
+++ lucene/dev/trunk/solr/src/test/test-files/solr/conf/schema-copyfield-test.xml Fri May
14 20:58:29 2010
@@ -394,6 +394,7 @@
       termVectors="true" termPositions="true" termOffsets="true"/>
    <field name="highlight"  type="text" indexed="true"  stored="true" 
       termVectors="true" termPositions="true" termOffsets="true"/>
+   <field name="signatureField" type="string" indexed="true" stored="false"/>
    
 
    

Modified: lucene/dev/trunk/solr/src/test/test-files/solr/conf/schema-luceneMatchVersion.xml
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/src/test/test-files/solr/conf/schema-luceneMatchVersion.xml?rev=944463&r1=944462&r2=944463&view=diff
==============================================================================
--- lucene/dev/trunk/solr/src/test/test-files/solr/conf/schema-luceneMatchVersion.xml (original)
+++ lucene/dev/trunk/solr/src/test/test-files/solr/conf/schema-luceneMatchVersion.xml Fri
May 14 20:58:29 2010
@@ -17,6 +17,7 @@
 -->
 <schema name="luceneMatchVersionTest" version="1.1">
  <types>
+  <fieldtype name="string" class="solr.StrField"/>
   <fieldtype name="text20" class="solr.TextField">
     <analyzer>
       <tokenizer class="solr.StandardTokenizerFactory" luceneMatchVersion="LUCENE_20"/>
@@ -43,9 +44,11 @@
   </fieldtype>
  </types>
  <fields>
+   <field name="signatureField" type="string" indexed="true" stored="false"/>
    <field name="text20" type="text20" indexed="true" stored="false" />
    <field name="textDefault" type="textDefault" indexed="true" stored="false" />
    <field name="textStandardAnalyzer20" type="textStandardAnalyzer20" indexed="true" stored="false"
/>
    <field name="textStandardAnalyzerDefault" type="textStandardAnalyzerDefault" indexed="true"
stored="false" />
+   <dynamicField name="*_sS" type="string"  indexed="false" stored="true"/>
  </fields>
 </schema>

Modified: lucene/dev/trunk/solr/src/test/test-files/solr/conf/schema-not-required-unique-key.xml
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/src/test/test-files/solr/conf/schema-not-required-unique-key.xml?rev=944463&r1=944462&r2=944463&view=diff
==============================================================================
--- lucene/dev/trunk/solr/src/test/test-files/solr/conf/schema-not-required-unique-key.xml
(original)
+++ lucene/dev/trunk/solr/src/test/test-files/solr/conf/schema-not-required-unique-key.xml
Fri May 14 20:58:29 2010
@@ -39,6 +39,8 @@
  <fields>
    <field name="id" type="sint" indexed="true" stored="true" required="false"/>
    <field name="subject" type="text" indexed="true" stored="true"/>
+   <field name="signatureField" type="string" indexed="true" stored="false"/>
+   <dynamicField name="*_sS" type="string"  indexed="false" stored="true"/>
  </fields>
 
  <defaultSearchField>subject</defaultSearchField>

Modified: lucene/dev/trunk/solr/src/test/test-files/solr/conf/schema-required-fields.xml
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/src/test/test-files/solr/conf/schema-required-fields.xml?rev=944463&r1=944462&r2=944463&view=diff
==============================================================================
--- lucene/dev/trunk/solr/src/test/test-files/solr/conf/schema-required-fields.xml (original)
+++ lucene/dev/trunk/solr/src/test/test-files/solr/conf/schema-required-fields.xml Fri May
14 20:58:29 2010
@@ -376,6 +376,7 @@
    <field name="timestamp" type="date" indexed="true" stored="true" default="NOW" multiValued="false"/>
    <field name="multiDefault" type="string" indexed="true" stored="true" default="muLti-Default"
multiValued="true"/>
    <field name="intDefault" type="sint" indexed="true" stored="true" default="42" multiValued="false"/>
+   <field name="signatureField" type="string" indexed="true" stored="false"/>
    
 
    

Modified: lucene/dev/trunk/solr/src/test/test-files/solr/conf/schema-reversed.xml
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/src/test/test-files/solr/conf/schema-reversed.xml?rev=944463&r1=944462&r2=944463&view=diff
==============================================================================
--- lucene/dev/trunk/solr/src/test/test-files/solr/conf/schema-reversed.xml (original)
+++ lucene/dev/trunk/solr/src/test/test-files/solr/conf/schema-reversed.xml Fri May 14 20:58:29
2010
@@ -26,6 +26,7 @@
 
 
     <fieldtype name="integer" class="solr.IntField" />
+    <fieldtype name="string" class="solr.StrField" />
 
     <fieldtype name="text" class="solr.TextField">
       <analyzer>
@@ -72,6 +73,9 @@
    <field name="two" type="rev" indexed="true" stored="false"/>
    <field name="three" type="text" indexed="true" stored="false"/>
 
+   <field name="signatureField" type="string" indexed="true" stored="false"/>
+   <dynamicField name="*_sS" type="string"  indexed="false" stored="true"/>
+
  </fields>
 
  <defaultSearchField>one</defaultSearchField>

Modified: lucene/dev/trunk/solr/src/test/test-files/solr/conf/schema-trie.xml
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/src/test/test-files/solr/conf/schema-trie.xml?rev=944463&r1=944462&r2=944463&view=diff
==============================================================================
--- lucene/dev/trunk/solr/src/test/test-files/solr/conf/schema-trie.xml (original)
+++ lucene/dev/trunk/solr/src/test/test-files/solr/conf/schema-trie.xml Fri May 14 20:58:29
2010
@@ -274,6 +274,7 @@
    <!-- for testing, a type that does a transform to see if it's correctly done everywhere
-->
    <field name="id" type="sfloat" indexed="true" stored="true" required="true" />
    <field name="text" type="text" indexed="true" stored="false" />
+   <field name="signatureField" type="string" indexed="true" stored="false"/>
 
    <field name="tint" type="tint" indexed="true" stored="true" />
    <field name="tfloat" type="tfloat" indexed="true" stored="true" />
@@ -293,6 +294,7 @@
         both match, the first appearing in the schema will be used.  -->
    <dynamicField name="*_s"  type="string"  indexed="true"  stored="true"/>
    <dynamicField name="*_ss"  type="string"  indexed="true"  stored="true" multiValued="true"/>
+   <dynamicField name="*_sS" type="string"  indexed="false" stored="true"/>
    <dynamicField name="*_i"  type="sint"    indexed="true"  stored="true"/>
    <dynamicField name="*_l"  type="slong"   indexed="true"  stored="true"/>
    <dynamicField name="*_f"  type="sfloat"  indexed="true"  stored="true"/>

Modified: lucene/dev/trunk/solr/src/test/test-files/solr/conf/schema11.xml
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/src/test/test-files/solr/conf/schema11.xml?rev=944463&r1=944462&r2=944463&view=diff
==============================================================================
--- lucene/dev/trunk/solr/src/test/test-files/solr/conf/schema11.xml (original)
+++ lucene/dev/trunk/solr/src/test/test-files/solr/conf/schema11.xml Fri May 14 20:58:29 2010
@@ -292,6 +292,8 @@
 
    <field name="point_hash" type="geohash" indexed="true" stored="true" multiValued="false"/>
 
+   <field name="signatureField" type="string" indexed="true" stored="false"/>
+
    <!-- Dynamic field definitions.  If a field name is not found, dynamicFields
         will be used if the name matches any of the patterns.
         RESTRICTION: the glob-like pattern in the name attribute must have
@@ -301,6 +303,7 @@
         both match, the first appearing in the schema will be used.  -->
    <dynamicField name="*_s"  type="string"  indexed="true"  stored="true"/>
    <dynamicField name="*_ss"  type="string"  indexed="true"  stored="true" multiValued="true"/>
+   <dynamicField name="*_sS" type="string"  indexed="false" stored="true"/>
    <dynamicField name="*_ii"  type="integer"    indexed="true"  stored="true" multiValued="true"/>
    <dynamicField name="*_i"  type="sint"    indexed="true"  stored="true"/>
    <dynamicField name="*_is"  type="sint"    indexed="true"  stored="true" multiValued="true"/>

Modified: lucene/dev/trunk/solr/src/test/test-files/solr/conf/solrconfig.xml
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/src/test/test-files/solr/conf/solrconfig.xml?rev=944463&r1=944462&r2=944463&view=diff
==============================================================================
--- lucene/dev/trunk/solr/src/test/test-files/solr/conf/solrconfig.xml (original)
+++ lucene/dev/trunk/solr/src/test/test-files/solr/conf/solrconfig.xml Fri May 14 20:58:29
2010
@@ -459,5 +459,18 @@
     </processor>
     <processor class="solr.RunUpdateProcessorFactory" />
   </updateRequestProcessorChain>
-
+  <updateRequestProcessorChain name="stored_sig">
+    <!-- this chain is valid even though the signature field is not
+         indexed, because we are not asking for dups to be overwritten
+      -->
+    <processor class="org.apache.solr.update.processor.SignatureUpdateProcessorFactory">
+      <bool name="enabled">true</bool>
+      <str name="signatureField">non_indexed_signature_sS</str>
+      <bool name="overwriteDupes">false</bool>
+      <str name="fields">v_t,t_field</str>
+      <str name="signatureClass">org.apache.solr.update.processor.TextProfileSignature</str>
+    </processor>
+    <processor class="solr.RunUpdateProcessorFactory" />
+  </updateRequestProcessorChain>
+  
 </config>



Mime
View raw message