manifoldcf-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mlizew...@apache.org
Subject svn commit: r1499078 - in /manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr: HttpPoster.java SolrConnector.java
Date Tue, 02 Jul 2013 19:54:52 GMT
Author: mlizewski
Date: Tue Jul  2 19:54:52 2013
New Revision: 1499078

URL: http://svn.apache.org/r1499078
Log:
improvement CONNECTORS-746: allow multiple field mappings in Solr output connector

Modified:
    manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java
    manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConnector.java

Modified: manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java?rev=1499078&r1=1499077&r2=1499078&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java
(original)
+++ manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java
Tue Jul  2 19:54:52 2013
@@ -491,7 +491,7 @@ public class HttpPoster
   * @throws ManifoldCFException, ServiceInterruption
   */
   public boolean indexPost(String documentURI,
-    RepositoryDocument document, Map arguments, Map sourceTargets,
+    RepositoryDocument document, Map arguments, Map<String, List<String>> sourceTargets,
     String authorityNameString, IOutputAddActivity activities)
     throws ManifoldCFException, ServiceInterruption
   {
@@ -767,7 +767,7 @@ public class HttpPoster
     protected String documentURI;
     protected RepositoryDocument document;
     protected Map<String,List<String>> arguments;
-    protected Map<String,String> sourceTargets;
+    protected Map<String,List<String>> sourceTargets;
     protected String[] shareAcls;
     protected String[] shareDenyAcls;
     protected String[] acls;
@@ -783,7 +783,7 @@ public class HttpPoster
     protected boolean rval = false;
 
     public IngestThread(String documentURI, RepositoryDocument document,
-      Map<String,List<String>> arguments, Map<String,String> sourceTargets,
+      Map<String,List<String>> arguments, Map<String, List<String>>
sourceTargets,
       String[] shareAcls, String[] shareDenyAcls, String[] acls, String[] denyAcls, String
commitWithin)
     {
       super();
@@ -871,15 +871,26 @@ public class HttpPoster
           while (iter.hasNext())
           {
             String fieldName = iter.next();
-            String newFieldName = sourceTargets.get(fieldName);
-            if (newFieldName == null)
-              newFieldName = fieldName;
-            if (newFieldName.length() > 0)
-            {
-              if (newFieldName.toLowerCase(Locale.ROOT).equals(idAttributeName.toLowerCase(Locale.ROOT)))
-                newFieldName = ID_METADATA;
-              String[] values = document.getFieldAsStrings(fieldName);
-              writeField(out,LITERAL+newFieldName,values);
+            List<String> mapping = sourceTargets.get(fieldName);
+            if(mapping != null) {
+              for(String newFieldName : mapping) {
+                if(newFieldName != null && !newFieldName.isEmpty()) {
+                  if (newFieldName.toLowerCase(Locale.ROOT).equals(idAttributeName.toLowerCase(Locale.ROOT)))
{
+                    newFieldName = ID_METADATA;
+                  }
+                  String[] values = document.getFieldAsStrings(fieldName);
+                  writeField(out,LITERAL+newFieldName,values);
+                }
+              }
+            } else {
+              String newFieldName = fieldName;
+              if (!newFieldName.isEmpty()) {
+                if (newFieldName.toLowerCase(Locale.ROOT).equals(idAttributeName.toLowerCase(Locale.ROOT)))
{
+                  newFieldName = ID_METADATA;
+                }
+                String[] values = document.getFieldAsStrings(fieldName);
+                writeField(out,LITERAL+newFieldName,values);
+              }
             }
           }
              

Modified: manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConnector.java?rev=1499078&r1=1499077&r2=1499078&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConnector.java
(original)
+++ manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConnector.java
Tue Jul  2 19:54:52 2013
@@ -463,6 +463,7 @@ public class SolrConnector extends org.a
     {
       String name = sortArray[i++];
       ArrayList values = (ArrayList)args.get(name);
+      java.util.Collections.sort(values);
       int j = 0;
       while (j < values.size())
       {
@@ -477,45 +478,52 @@ public class SolrConnector extends org.a
     
     packList(sb,nameValues,'+');
     
-    Map fieldMap = new HashMap();
+    // Do the source/target pairs
     i = 0;
-    while (i < spec.getChildCount())
-    {
+    Map<String, List<String>> sourceTargets = new HashMap<String, List<String>>();
+    while (i < spec.getChildCount()) {
       SpecificationNode sn = spec.getChild(i++);
-      if (sn.getType().equals(SolrConfig.NODE_FIELDMAP))
-      {
+      if (sn.getType().equals(SolrConfig.NODE_FIELDMAP)) {
         String source = sn.getAttributeValue(SolrConfig.ATTRIBUTE_SOURCE);
         String target = sn.getAttributeValue(SolrConfig.ATTRIBUTE_TARGET);
-        if (target == null)
+        if (target == null) {
           target = "";
-        fieldMap.put(source,target);
+        }
+        List<String> list = (List<String>)sourceTargets.get(source);
+        if (list == null) {
+          list = new ArrayList<String>();
+          sourceTargets.put(source, list);
+        }
+        list.add(target);
       }
     }
     
-    sortArray = new String[fieldMap.size()];
+    sortArray = new String[sourceTargets.size()];
+    iter = sourceTargets.keySet().iterator();
     i = 0;
-    iter = fieldMap.keySet().iterator();
-    while (iter.hasNext())
-    {
+    while (iter.hasNext()) {
       sortArray[i++] = (String)iter.next();
     }
     java.util.Arrays.sort(sortArray);
     
-    ArrayList sourceTargets = new ArrayList();
-    
+    ArrayList sourceTargetsList = new ArrayList();
     i = 0;
-    while (i < sortArray.length)
-    {
+    while (i < sortArray.length) {
       String source = sortArray[i++];
-      String target = (String)fieldMap.get(source);
-      fixedList[0] = source;
-      fixedList[1] = target;
-      StringBuilder pairBuffer = new StringBuilder();
-      packFixedList(pairBuffer,fixedList,'=');
-      sourceTargets.add(pairBuffer.toString());
+      List<String> values = (List<String>)sourceTargets.get(source);
+      java.util.Collections.sort(values);
+      int j = 0;
+      while (j < values.size()) {
+        String target = (String)values.get(j++);
+        fixedList[0] = source;
+        fixedList[1] = target;
+        StringBuilder pairBuffer = new StringBuilder();
+        packFixedList(pairBuffer,fixedList,'=');
+        sourceTargetsList.add(pairBuffer.toString());
+      }
     }
     
-    packList(sb,sourceTargets,'+');
+    packList(sb,sourceTargetsList,'+');
 
     // Here, append things which we have no intention of unpacking.  This includes stuff
that comes from
     // the configuration information, for instance.
@@ -601,7 +609,7 @@ public class SolrConnector extends org.a
   {
     // Build the argument map we'll send.
     Map args = new HashMap();
-    Map sourceTargets = new HashMap();
+    Map<String, List<String>> sourceTargets = new HashMap<String, List<String>>();
     int index = 0;
     ArrayList nameValues = new ArrayList();
     index = unpackList(nameValues,outputDescription,index,'+');
@@ -627,11 +635,17 @@ public class SolrConnector extends org.a
     
     // Do the source/target pairs
     i = 0;
-    while (i < sts.size())
-    {
+    while (i < sts.size()) {
       String x = (String)sts.get(i++);
       unpackFixedList(fixedBuffer,x,0,'=');
-      sourceTargets.put(fixedBuffer[0],fixedBuffer[1]);
+      String source = fixedBuffer[0];
+      String target = fixedBuffer[1];
+      List<String> list = (List<String>)sourceTargets.get(source);
+      if (list == null) {
+        list = new ArrayList<String>();
+        sourceTargets.put(source, list);
+      }
+      list.add(target);
     }
 
     // Establish a session
@@ -2238,21 +2252,7 @@ public class SolrConnector extends org.a
   public void outputSpecificationBody(IHTTPOutput out, Locale locale, OutputSpecification
os, String tabName)
     throws ManifoldCFException, IOException
   {
-    // Prep for field mapping tab
-    HashMap fieldMap = new HashMap();
     int i = 0;
-    while (i < os.getChildCount())
-    {
-      SpecificationNode sn = os.getChild(i++);
-      if (sn.getType().equals(SolrConfig.NODE_FIELDMAP))
-      {
-        String source = sn.getAttributeValue(SolrConfig.ATTRIBUTE_SOURCE);
-        String target = sn.getAttributeValue(SolrConfig.ATTRIBUTE_TARGET);
-        if (target != null && target.length() == 0)
-          target = null;
-        fieldMap.put(source,target);
-      }
-    }
     
     // Field Mapping tab
     if (tabName.equals(Messages.getString(locale,"SolrConnector.SolrFieldMapping")))
@@ -2271,30 +2271,25 @@ public class SolrConnector extends org.a
 "        </tr>\n"
       );
 
-      String[] sourceFieldNames = new String[fieldMap.size()];
-      Iterator iter = fieldMap.keySet().iterator();
-      i = 0;
-      while (iter.hasNext())
-      {
-        sourceFieldNames[i++] = (String)iter.next();
-      }
-      java.util.Arrays.sort(sourceFieldNames);
-      
       int fieldCounter = 0;
       i = 0;
-      while (i < sourceFieldNames.length)
-      {
-        String source = sourceFieldNames[i++];
-        String target = (String)fieldMap.get(source);
-        String targetDisplay = target;
-        if (target == null)
-        {
-          target = "";
-          targetDisplay = "(remove)";
-        }
-        // It's prefix will be...
-        String prefix = "solr_fieldmapping_" + Integer.toString(fieldCounter);
-        out.print(
+      while (i < os.getChildCount()) {
+        SpecificationNode sn = os.getChild(i++);
+        if (sn.getType().equals(SolrConfig.NODE_FIELDMAP)) {
+          String source = sn.getAttributeValue(SolrConfig.ATTRIBUTE_SOURCE);
+          String target = sn.getAttributeValue(SolrConfig.ATTRIBUTE_TARGET);
+          if (target != null && target.length() == 0) {
+            target = null;
+          }
+          String targetDisplay = target;
+          if (target == null)
+          {
+            target = "";
+            targetDisplay = "(remove)";
+          }
+          // It's prefix will be...
+          String prefix = "solr_fieldmapping_" + Integer.toString(fieldCounter);
+          out.print(
 "        <tr class=\""+(((fieldCounter % 2)==0)?"evenformrow":"oddformrow")+"\">\n"+
 "          <td class=\"formcolumncell\">\n"+
 "            <a name=\""+prefix+"\">\n"+
@@ -2311,8 +2306,9 @@ public class SolrConnector extends org.a
 "            <nobr>"+org.apache.manifoldcf.ui.util.Encoder.bodyEscape(targetDisplay)+"</nobr>\n"+
 "          </td>\n"+
 "        </tr>\n"
-        );
-        fieldCounter++;
+          );
+          fieldCounter++;
+        }
       }
       
       if (fieldCounter == 0)
@@ -2347,25 +2343,27 @@ public class SolrConnector extends org.a
     else
     {
       // Hiddens for field mapping
-      out.print(
-"<input type=\"hidden\" name=\"solr_fieldmapping_count\" value=\""+Integer.toString(fieldMap.size())+"\"/>\n"
-      );
-      Iterator iter = fieldMap.keySet().iterator();
+      i = 0;
       int fieldCounter = 0;
-      while (iter.hasNext())
-      {
-        String source = (String)iter.next();
-        String target = (String)fieldMap.get(source);
-        if (target == null)
-          target = "";
+      while (i < os.getChildCount()) {
+        SpecificationNode sn = os.getChild(i++);
+        if (sn.getType().equals(SolrConfig.NODE_FIELDMAP)) {
+          String source = sn.getAttributeValue(SolrConfig.ATTRIBUTE_SOURCE);
+          String target = sn.getAttributeValue(SolrConfig.ATTRIBUTE_TARGET);
+          if (target == null)
+            target = "";
         // It's prefix will be...
-        String prefix = "solr_fieldmapping_" + Integer.toString(fieldCounter);
-        out.print(
+          String prefix = "solr_fieldmapping_" + Integer.toString(fieldCounter);
+          out.print(
 "<input type=\"hidden\" name=\""+prefix+"_source\" value=\""+org.apache.manifoldcf.ui.util.Encoder.attributeEscape(source)+"\"/>\n"+
 "<input type=\"hidden\" name=\""+prefix+"_target\" value=\""+org.apache.manifoldcf.ui.util.Encoder.attributeEscape(target)+"\"/>\n"
-        );
-        fieldCounter++;
+          );
+          fieldCounter++;
+        }
       }
+      out.print(
+"<input type=\"hidden\" name=\"solr_fieldmapping_count\" value=\""+Integer.toString(fieldCounter)+"\"/>\n"
+      );
     }
 
   }
@@ -2444,27 +2442,6 @@ public class SolrConnector extends org.a
     // Prep for field mappings
     HashMap fieldMap = new HashMap();
     int i = 0;
-    while (i < os.getChildCount())
-    {
-      SpecificationNode sn = os.getChild(i++);
-      if (sn.getType().equals(SolrConfig.NODE_FIELDMAP))
-      {
-        String source = sn.getAttributeValue(SolrConfig.ATTRIBUTE_SOURCE);
-        String target = sn.getAttributeValue(SolrConfig.ATTRIBUTE_TARGET);
-        if (target != null && target.length() == 0)
-          target = null;
-        fieldMap.put(source,target);
-      }
-    }
-
-    String[] sourceFieldNames = new String[fieldMap.size()];
-    Iterator iter = fieldMap.keySet().iterator();
-    i = 0;
-    while (iter.hasNext())
-    {
-      sourceFieldNames[i++] = (String)iter.next();
-    }
-    java.util.Arrays.sort(sourceFieldNames);
 
     // Display field mappings
     out.print(
@@ -2481,17 +2458,19 @@ public class SolrConnector extends org.a
     );
 
     int fieldCounter = 0;
-    while (fieldCounter < sourceFieldNames.length)
-    {
-      String source = sourceFieldNames[fieldCounter++];
-      String target = (String)fieldMap.get(source);
-      String targetDisplay = target;
-      if (target == null)
-      {
-        target = "";
-        targetDisplay = "(remove)";
-      }
-      out.print(
+    i = 0;
+    while (i < os.getChildCount()) {
+      SpecificationNode sn = os.getChild(i++);
+      if (sn.getType().equals(SolrConfig.NODE_FIELDMAP)) {
+        String source = sn.getAttributeValue(SolrConfig.ATTRIBUTE_SOURCE);
+        String target = sn.getAttributeValue(SolrConfig.ATTRIBUTE_TARGET);
+        String targetDisplay = target;
+        if (target == null)
+        {
+          target = "";
+          targetDisplay = "(remove)";
+        }
+        out.print(
 "        <tr class=\""+(((fieldCounter % 2)==0)?"evenformrow":"oddformrow")+"\">\n"+
 "          <td class=\"formcolumncell\">\n"+
 "            <nobr>"+org.apache.manifoldcf.ui.util.Encoder.bodyEscape(source)+"</nobr>\n"+
@@ -2500,8 +2479,9 @@ public class SolrConnector extends org.a
 "            <nobr>"+org.apache.manifoldcf.ui.util.Encoder.bodyEscape(targetDisplay)+"</nobr>\n"+
 "          </td>\n"+
 "        </tr>\n"
-      );
-      fieldCounter++;
+        );
+        fieldCounter++;
+      }
     }
     
     if (fieldCounter == 0)



Mime
View raw message