lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From arafa...@apache.org
Subject lucene-solr:branch_7x: SOLR-2834: Handle CharacterFilters in Solr Impacts Document and Field Analyzes and was previously a class cast error
Date Thu, 05 Jul 2018 02:21:07 GMT
Repository: lucene-solr
Updated Branches:
  refs/heads/branch_7x 7fb97a222 -> 6c3f91d35


SOLR-2834: Handle CharacterFilters in Solr
Impacts Document and Field Analyzes and was previously a class cast error


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/6c3f91d3
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/6c3f91d3
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/6c3f91d3

Branch: refs/heads/branch_7x
Commit: 6c3f91d35ea0d767e7f945fbf5c132d1aaac17e4
Parents: 7fb97a2
Author: Alexandre Rafalovitch <arafalov@apache.org>
Authored: Wed Jul 4 22:19:54 2018 -0400
Committer: Alexandre Rafalovitch <arafalov@apache.org>
Committed: Wed Jul 4 22:19:54 2018 -0400

----------------------------------------------------------------------
 solr/CHANGES.txt                                |  3 ++
 .../solrj/response/AnalysisResponseBase.java    | 29 ++++++++++++++++----
 .../response/DocumentAnalysisResponse.java      | 12 ++++----
 .../solrj/response/FieldAnalysisResponse.java   | 14 +++++-----
 .../solrj/response/AnlysisResponseBaseTest.java | 13 +++++++++
 .../response/DocumentAnalysisResponseTest.java  |  2 +-
 .../response/FieldAnalysisResponseTest.java     |  2 +-
 7 files changed, 55 insertions(+), 20 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6c3f91d3/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 4b4ba24..e46affe 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -109,6 +109,9 @@ Bug Fixes
 
 * SOLR-12533 Collection collection fails if metrics are called during core creation (Peter
Cseh, Mano Kovacs)
 
+* SOLR-2834: Fix SolrJ Field and Document analyzes for types that include CharacterFilter
(Alexandre Rafalovitch)
+
+
 Optimizations
 ----------------------
 

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6c3f91d3/solr/solrj/src/java/org/apache/solr/client/solrj/response/AnalysisResponseBase.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/response/AnalysisResponseBase.java
b/solr/solrj/src/java/org/apache/solr/client/solrj/response/AnalysisResponseBase.java
index b2d1884..5416376 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/response/AnalysisResponseBase.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/response/AnalysisResponseBase.java
@@ -57,18 +57,28 @@ public class AnalysisResponseBase extends SolrResponseBase {
    *  &lt;/lst&gt;
    * </code></pre>
    *
+   * The special case is a CharacterFilter that just returns a string, which we then map
to a single token without type.
+   *
    * @param phaseNL The names list to parse.
    *
    * @return The built analysis phases list.
    */
-  protected List<AnalysisPhase> buildPhases(NamedList<List<NamedList<Object>>>
phaseNL) {
+  protected List<AnalysisPhase> buildPhases(NamedList<Object> phaseNL) {
     List<AnalysisPhase> phases = new ArrayList<>(phaseNL.size());
-    for (Map.Entry<String, List<NamedList<Object>>> phaseEntry : phaseNL)
{
+    for (Map.Entry<String, Object> phaseEntry : phaseNL) {
       AnalysisPhase phase = new AnalysisPhase(phaseEntry.getKey());
-      List<NamedList<Object>> tokens = phaseEntry.getValue();
-      for (NamedList<Object> token : tokens) {
-        TokenInfo tokenInfo = buildTokenInfo(token);
+      Object phaseValue = phaseEntry.getValue();
+
+      if (phaseValue instanceof String) {
+        // We are looking at CharacterFilter, which - exceptionally - returns a string
+        TokenInfo tokenInfo = buildTokenInfoFromString((String) phaseValue);
         phase.addTokenInfo(tokenInfo);
+      } else {
+        List<NamedList<Object>> tokens = (List<NamedList<Object>>)
phaseEntry.getValue();
+        for (NamedList<Object> token : tokens) {
+          TokenInfo tokenInfo = buildTokenInfo(token);
+          phase.addTokenInfo(tokenInfo);
+        }
       }
       phases.add(phase);
     }
@@ -76,6 +86,15 @@ public class AnalysisResponseBase extends SolrResponseBase {
   }
 
   /**
+   * Convert a string value (from CharacterFilter) into a TokenInfo for its value full span.
+   * @param value String value
+   * @return The built token info (with type set to null)
+   */
+  protected TokenInfo buildTokenInfoFromString(String value) {
+    return new TokenInfo(value, value, null, 0, value.length(), 1, false);
+  }
+
+  /**
    * Parses the given named list and builds a token infoform it. Expects a named list of
the form:
    * <br>
    * <pre><code>

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6c3f91d3/solr/solrj/src/java/org/apache/solr/client/solrj/response/DocumentAnalysisResponse.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/response/DocumentAnalysisResponse.java
b/solr/solrj/src/java/org/apache/solr/client/solrj/response/DocumentAnalysisResponse.java
index 231199d..311318e 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/response/DocumentAnalysisResponse.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/response/DocumentAnalysisResponse.java
@@ -49,19 +49,19 @@ public class DocumentAnalysisResponse extends AnalysisResponseBase implements
It
         NamedList<Object> field = fieldEntry.getValue();
 
         @SuppressWarnings("unchecked")
-        NamedList<List<NamedList<Object>>> query 
-          = (NamedList<List<NamedList<Object>>>) field.get("query");
+        NamedList<Object> query
+          = (NamedList<Object>) field.get("query");
         if (query != null) {
           List<AnalysisPhase> phases = buildPhases(query);
           fieldAnalysis.setQueryPhases(phases);
         }
         
         @SuppressWarnings("unchecked")
-        NamedList<NamedList<List<NamedList<Object>>>> index 
-          = (NamedList<NamedList<List<NamedList<Object>>>>) field.get("index");
-        for (Map.Entry<String, NamedList<List<NamedList<Object>>>>
valueEntry : index) {
+        NamedList<NamedList<Object>> index
+          = (NamedList<NamedList<Object>>) field.get("index");
+        for (Map.Entry<String, NamedList<Object>> valueEntry : index) {
           String fieldValue = valueEntry.getKey();
-          NamedList<List<NamedList<Object>>> valueNL = valueEntry.getValue();
+          NamedList<Object> valueNL = valueEntry.getValue();
           List<AnalysisPhase> phases = buildPhases(valueNL);
           fieldAnalysis.setIndexPhases(fieldValue, phases);
         }

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6c3f91d3/solr/solrj/src/java/org/apache/solr/client/solrj/response/FieldAnalysisResponse.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/response/FieldAnalysisResponse.java
b/solr/solrj/src/java/org/apache/solr/client/solrj/response/FieldAnalysisResponse.java
index 533885b..032af0c1 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/response/FieldAnalysisResponse.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/response/FieldAnalysisResponse.java
@@ -39,30 +39,30 @@ public class FieldAnalysisResponse extends AnalysisResponseBase {
     super.setResponse(response);
 
     @SuppressWarnings("unchecked")
-    NamedList<NamedList<NamedList<NamedList<List<NamedList<Object>>>>>>
analysisNL 
-      = (NamedList<NamedList<NamedList<NamedList<List<NamedList<Object>>>>>>)
response.get("analysis");
+    NamedList<NamedList<NamedList<NamedList<Object>>>> analysisNL
+      = (NamedList<NamedList<NamedList<NamedList<Object>>>>) response.get("analysis");
 
-    for (Map.Entry<String, NamedList<NamedList<List<NamedList<Object>>>>>
entry 
+    for (Map.Entry<String, NamedList<NamedList<Object>>> entry
            : analysisNL.get("field_types")) {
 
       analysisByFieldTypeName.put(entry.getKey(), buildAnalysis(entry.getValue()));
     }
 
-    for (Map.Entry<String, NamedList<NamedList<List<NamedList<Object>>>>>
entry 
+    for (Map.Entry<String, NamedList<NamedList<Object>>> entry
            : analysisNL.get("field_names")) {
 
       analysisByFieldName.put(entry.getKey(), buildAnalysis(entry.getValue()));
     }
   }
 
-  private Analysis buildAnalysis(NamedList<NamedList<List<NamedList<Object>>>>
value) {
+  private Analysis buildAnalysis(NamedList<NamedList<Object>> value) {
       Analysis analysis = new Analysis();
       
-      NamedList<List<NamedList<Object>>> queryNL = value.get("query");
+      NamedList<Object> queryNL = value.get("query");
       List<AnalysisPhase> phases = (queryNL == null) ? null : buildPhases(queryNL);
       analysis.setQueryPhases(phases);
 
-      NamedList<List<NamedList<Object>>> indexNL = value.get("index");
+      NamedList<Object> indexNL = value.get("index");
       phases = buildPhases(indexNL);
       analysis.setIndexPhases(phases);
       

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6c3f91d3/solr/solrj/src/test/org/apache/solr/client/solrj/response/AnlysisResponseBaseTest.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/response/AnlysisResponseBaseTest.java
b/solr/solrj/src/test/org/apache/solr/client/solrj/response/AnlysisResponseBaseTest.java
index 228bf82..c168666 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/response/AnlysisResponseBaseTest.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/response/AnlysisResponseBaseTest.java
@@ -98,6 +98,19 @@ public class AnlysisResponseBaseTest extends LuceneTestCase {
     assertPhase(phases.get(3), "Filter3", 3, tokenInfo);
   }
 
+  /**
+   * Tests the {@link AnalysisResponseBase#buildPhases(org.apache.solr.common.util.NamedList)}
)}
+   * method for the special case of CharacterFilter.
+   */
+  @Test
+  public void testCharFilterBuildPhases() throws Exception {
+    NamedList nl = new NamedList();
+    nl.add("CharFilter1", "CharFilterOutput"); //not list of tokens
+    AnalysisResponseBase response = new AnalysisResponseBase();
+    List<AnalysisResponseBase.AnalysisPhase> phases = response.buildPhases(nl);
+    assertEquals(1, phases.size());
+  }
+
   //================================================ Helper Methods ==================================================
 
   private List<NamedList> buildFakeTokenInfoList(int numberOfTokens) {

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6c3f91d3/solr/solrj/src/test/org/apache/solr/client/solrj/response/DocumentAnalysisResponseTest.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/response/DocumentAnalysisResponseTest.java
b/solr/solrj/src/test/org/apache/solr/client/solrj/response/DocumentAnalysisResponseTest.java
index 2a60e94..6e4b785 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/response/DocumentAnalysisResponseTest.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/response/DocumentAnalysisResponseTest.java
@@ -47,7 +47,7 @@ public class DocumentAnalysisResponseTest extends LuceneTestCase {
     DocumentAnalysisResponse response = new DocumentAnalysisResponse() {
 
       @Override
-      protected List<AnalysisPhase> buildPhases(NamedList<List<NamedList<Object>>>
phaseNL) {
+      protected List<AnalysisPhase> buildPhases(NamedList<Object> phaseNL) {
         return phases;
       }
     };

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6c3f91d3/solr/solrj/src/test/org/apache/solr/client/solrj/response/FieldAnalysisResponseTest.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/response/FieldAnalysisResponseTest.java
b/solr/solrj/src/test/org/apache/solr/client/solrj/response/FieldAnalysisResponseTest.java
index ee9c532..866aad1 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/response/FieldAnalysisResponseTest.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/response/FieldAnalysisResponseTest.java
@@ -48,7 +48,7 @@ public class FieldAnalysisResponseTest extends LuceneTestCase {
     NamedList responseNL = buildResponse();
     FieldAnalysisResponse response = new FieldAnalysisResponse() {
       @Override
-      protected List<AnalysisPhase> buildPhases(NamedList<List<NamedList<Object>>>
phaseNL) {
+      protected List<AnalysisPhase> buildPhases(NamedList<Object> phaseNL) {
         return phases;
       }
     };


Mime
View raw message