manifoldcf-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kwri...@apache.org
Subject svn commit: r1556668 - in /manifoldcf/branches/CONNECTORS-840/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr: HttpPoster.java SolrConnector.java
Date Wed, 08 Jan 2014 23:24:47 GMT
Author: kwright
Date: Wed Jan  8 23:24:46 2014
New Revision: 1556668

URL: http://svn.apache.org/r1556668
Log:
Fix numerous problems with interpreting, packing, and unpacking new value

Modified:
    manifoldcf/branches/CONNECTORS-840/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java
    manifoldcf/branches/CONNECTORS-840/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConnector.java

Modified: manifoldcf/branches/CONNECTORS-840/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-840/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java?rev=1556668&r1=1556667&r2=1556668&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-840/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java
(original)
+++ manifoldcf/branches/CONNECTORS-840/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java
Wed Jan  8 23:24:46 2014
@@ -481,8 +481,8 @@ public class HttpPoster
   * @throws ManifoldCFException, ServiceInterruption
   */
   public boolean indexPost(String documentURI,
-                           RepositoryDocument document, Map arguments, Map<String, List<String>>
sourceTargets,
-                           Boolean keepAllMetadata, String authorityNameString, IOutputAddActivity
activities)
+    RepositoryDocument document, Map arguments, Map<String, List<String>> sourceTargets,
+    boolean keepAllMetadata, String authorityNameString, IOutputAddActivity activities)
     throws ManifoldCFException, ServiceInterruption
   {
     if (Logging.ingest.isDebugEnabled())
@@ -754,16 +754,16 @@ public class HttpPoster
   */
   protected class IngestThread extends java.lang.Thread
   {
-    protected String documentURI;
-    protected RepositoryDocument document;
-    protected Map<String,List<String>> arguments;
-    protected Map<String,List<String>> sourceTargets;
-    protected String[] shareAcls;
-    protected String[] shareDenyAcls;
-    protected String[] acls;
-    protected String[] denyAcls;
-    protected String commitWithin;
-    protected Boolean keepAllMetadata;
+    protected final String documentURI;
+    protected final RepositoryDocument document;
+    protected final Map<String,List<String>> arguments;
+    protected final Map<String,List<String>> sourceTargets;
+    protected final String[] shareAcls;
+    protected final String[] shareDenyAcls;
+    protected final String[] acls;
+    protected final String[] denyAcls;
+    protected final String commitWithin;
+    protected final boolean keepAllMetadata;
     
     protected Long activityStart = null;
     protected Long activityBytes = null;
@@ -773,14 +773,9 @@ public class HttpPoster
     protected boolean readFromDocumentStreamYet = false;
     protected boolean rval = false;
 
-      public IngestThread(String documentURI)
-      {
-          super();}
-
-
     public IngestThread(String documentURI, RepositoryDocument document,
-                        Map<String, List<String>> arguments, Boolean keepAllMetadata,
Map<String, List<String>> sourceTargets,
-                        String[] shareAcls, String[] shareDenyAcls, String[] acls, String[]
denyAcls, String commitWithin)
+      Map<String, List<String>> arguments, boolean keepAllMetadata, Map<String,
List<String>> sourceTargets,
+      String[] shareAcls, String[] shareDenyAcls, String[] acls, String[] denyAcls, String
commitWithin)
     {
       super();
       setDaemon(true);
@@ -796,15 +791,7 @@ public class HttpPoster
       this.keepAllMetadata=keepAllMetadata;
     }
 
-      public Boolean getKeepAllMetadata() {
-          return keepAllMetadata;
-      }
-
-      public void setKeepAllMetadata(Boolean keepAllMetadata) {
-          this.keepAllMetadata = keepAllMetadata;
-      }
-
-      public void run()
+    public void run()
     {
       long length = document.getBinaryLength();
       InputStream is = document.getBinaryStream();
@@ -962,50 +949,53 @@ public class HttpPoster
       }
     }
 
-      /**
-       * builds the solr parameter maps for the update request.
-       * For each mapping expressed is applied the renaming for the metadata field name.
-       * If we set to keep all the metadata, the metadata non present in the mapping will
be kept with their original names.
-       * In the other case ignored
-       * @param out
-       * @throws IOException
-       */
-        private void buildSolrParamsFromMetadata(ModifiableSolrParams out) throws IOException
-        {
-            if (this.keepAllMetadata)
-            {
-                Iterator<String> iter = document.getFields();
-                while (iter.hasNext())
-                {
-                    String fieldName = iter.next();
-                    List<String> mappings = sourceTargets.get(fieldName);
-                    if (mappings != null)
-                        for (String newFieldName : mappings)
-                            applySingleMapping(fieldName, out, newFieldName);
-                    else // the fields not mentioned in the mapping are added only if we
have set the keep all metadata=true.
-                        this.applySingleMapping(fieldName, out, fieldName);
-                }
-            }
-            else //don't keep all the metadata but only the ones in sourceTargets
-                for (String originalFieldName : sourceTargets.keySet())
-                {
-                    List<String> mapping = sourceTargets.get(originalFieldName);
-                    for (String newFieldName : mapping)
-                        applySingleMapping(originalFieldName, out, newFieldName);
-                }
-        }
-
-      private void applySingleMapping(String originalFieldName, ModifiableSolrParams out,
String newFieldName) throws IOException {
-          if(newFieldName != null && !newFieldName.isEmpty()) {
-              if (newFieldName.toLowerCase(Locale.ROOT).equals(idAttributeName.toLowerCase(Locale.ROOT)))
{
-                  newFieldName = ID_METADATA;
-              }
-              String[] values = document.getFieldAsStrings(originalFieldName);
-              writeField(out,LITERAL+newFieldName,values);
-          }
+    /**
+      * builds the solr parameter maps for the update request.
+      * For each mapping expressed is applied the renaming for the metadata field name.
+      * If we set to keep all the metadata, the metadata non present in the mapping will
be kept with their original names.
+      * In the other case ignored
+      * @param out
+      * @throws IOException
+      */
+    private void buildSolrParamsFromMetadata(ModifiableSolrParams out) throws IOException
+    {
+      if (this.keepAllMetadata)
+      {
+        Iterator<String> iter = document.getFields();
+        while (iter.hasNext())
+        {
+          String fieldName = iter.next();
+          List<String> mappings = sourceTargets.get(fieldName);
+          if (mappings != null)
+            for (String newFieldName : mappings)
+              applySingleMapping(fieldName, out, newFieldName);
+          else // the fields not mentioned in the mapping are added only if we have set the
keep all metadata=true.
+            applySingleMapping(fieldName, out, fieldName);
+        }
+      }
+      else
+      {
+        //don't keep all the metadata but only the ones in sourceTargets
+        for (String originalFieldName : sourceTargets.keySet())
+        {
+          List<String> mapping = sourceTargets.get(originalFieldName);
+          for (String newFieldName : mapping)
+            applySingleMapping(originalFieldName, out, newFieldName);
+        }
       }
+    }
+
+    private void applySingleMapping(String originalFieldName, ModifiableSolrParams out, String
newFieldName) throws IOException {
+      if(newFieldName != null && !newFieldName.isEmpty()) {
+        if (newFieldName.toLowerCase(Locale.ROOT).equals(idAttributeName.toLowerCase(Locale.ROOT)))
{
+          newFieldName = ID_METADATA;
+        }
+        String[] values = document.getFieldAsStrings(originalFieldName);
+        writeField(out,LITERAL+newFieldName,values);
+      }
+    }
 
-      public Throwable getException()
+    public Throwable getException()
     {
       return exception;
     }

Modified: manifoldcf/branches/CONNECTORS-840/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-840/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConnector.java?rev=1556668&r1=1556667&r2=1556668&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-840/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConnector.java
(original)
+++ manifoldcf/branches/CONNECTORS-840/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConnector.java
Wed Jan  8 23:24:46 2014
@@ -513,16 +513,14 @@ public class SolrConnector extends org.a
     // Do the source/target pairs
     i = 0;
     Map<String, List<String>> sourceTargets = new HashMap<String, List<String>>();
-    SpecificationNode keepAllMetadataNode = null;
+    boolean keepAllMetadata = true;
     while (i < spec.getChildCount()) {
       SpecificationNode sn = spec.getChild(i++);
       
       if(sn.getType().equals(SolrConfig.NODE_KEEPMETADATA)) {
-          keepAllMetadataNode = sn;
-          continue;
-      }
-      
-      if (sn.getType().equals(SolrConfig.NODE_FIELDMAP)) {
+        String value = sn.getAttributeValue(SolrConfig.ATTRIBUTE_VALUE);
+        keepAllMetadata = Boolean.parseBoolean(value);
+      } else if (sn.getType().equals(SolrConfig.NODE_FIELDMAP)) {
         String source = sn.getAttributeValue(SolrConfig.ATTRIBUTE_SOURCE);
         String target = sn.getAttributeValue(SolrConfig.ATTRIBUTE_TARGET);
         
@@ -565,6 +563,12 @@ public class SolrConnector extends org.a
     
     packList(sb,sourceTargetsList,'+');
 
+    // Keep all metadata flag
+    if (keepAllMetadata)
+      sb.append('+');
+    else
+      sb.append('-');
+
     // Here, append things which we have no intention of unpacking.  This includes stuff
that comes from
     // the configuration information, for instance.
     
@@ -596,17 +600,6 @@ public class SolrConnector extends org.a
         sb.append('-');
     }
     
-    // Add keepAllMetadata option if specified
-    if(keepAllMetadataNode != null) {
-        fixedList[0] = SolrConfig.NODE_KEEPMETADATA;
-        fixedList[1] = keepAllMetadataNode.getAttributeValue(SolrConfig.ATTRIBUTE_VALUE);
-        StringBuilder pairBuffer = new StringBuilder();
-        packFixedList(pairBuffer,fixedList,'=');
-        List<String> list = new ArrayList<String>();
-        list.add(pairBuffer.toString());
-        packList(sb, list,'+');
-    }
-    
     return sb.toString();
   }
 
@@ -663,7 +656,6 @@ public class SolrConnector extends org.a
     // Build the argument map we'll send.
     Map args = new HashMap();
     Map<String, List<String>> sourceTargets = new HashMap<String, List<String>>();
-    Boolean keepAllMetadata = false;
     int index = 0;
     ArrayList nameValues = new ArrayList();
     index = unpackList(nameValues,outputDescription,index,'+');
@@ -705,11 +697,12 @@ public class SolrConnector extends org.a
     }
 
     // extract keep all metadata Flag
-    String keepAllMetadataFlag=(String)metadataExtraParams.get(0);
-    unpackFixedList(fixedBuffer, keepAllMetadataFlag, 0, '=');
-    String keepAllMetadataFlagValue = fixedBuffer[1];
-    keepAllMetadata=Boolean.parseBoolean(keepAllMetadataFlagValue);
-
+    boolean keepAllMetadata = true;
+    if (index < outputDescription.length())
+    {
+      keepAllMetadata = (outputDescription.charAt(index++) == '+');
+    }
+    
     // Establish a session
     getSession();
 



Mime
View raw message