incubator-connectors-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kwri...@apache.org
Subject svn commit: r1132386 - in /incubator/lcf/trunk: ./ connectors/gts/connector/src/main/java/org/apache/manifoldcf/agents/output/gts/ connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/ framework/agents/src/main/java/org/apac...
Date Sun, 05 Jun 2011 12:36:31 GMT
Author: kwright
Date: Sun Jun  5 12:36:30 2011
New Revision: 1132386

URL: http://svn.apache.org/viewvc?rev=1132386&view=rev
Log:
Flesh out proper support for metadata readers.  CONNECTORS-208.

Modified:
    incubator/lcf/trunk/CHANGES.txt
    incubator/lcf/trunk/connectors/gts/connector/src/main/java/org/apache/manifoldcf/agents/output/gts/HttpPoster.java
    incubator/lcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java
    incubator/lcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/RepositoryDocument.java

Modified: incubator/lcf/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/CHANGES.txt?rev=1132386&r1=1132385&r2=1132386&view=diff
==============================================================================
--- incubator/lcf/trunk/CHANGES.txt (original)
+++ incubator/lcf/trunk/CHANGES.txt Sun Jun  5 12:36:30 2011
@@ -3,6 +3,10 @@ $Id$
 
 ======================= 0.3-dev =========================
 
+CONNECTORS-208: Make support for metadata Reader objects really
+work.
+(Karl Wright)
+
 CONNECTORS-204: Add an HSQLDB set of test targets, which use that
 database.
 (Karl Wright)

Modified: incubator/lcf/trunk/connectors/gts/connector/src/main/java/org/apache/manifoldcf/agents/output/gts/HttpPoster.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/connectors/gts/connector/src/main/java/org/apache/manifoldcf/agents/output/gts/HttpPoster.java?rev=1132386&r1=1132385&r2=1132386&view=diff
==============================================================================
--- incubator/lcf/trunk/connectors/gts/connector/src/main/java/org/apache/manifoldcf/agents/output/gts/HttpPoster.java
(original)
+++ incubator/lcf/trunk/connectors/gts/connector/src/main/java/org/apache/manifoldcf/agents/output/gts/HttpPoster.java
Sun Jun  5 12:36:30 2011
@@ -780,11 +780,11 @@ public class HttpPoster
                   }
 
                   // Write all the metadata, if any
-                  Iterator iter = document.getFields();
+                  Iterator<String> iter = document.getFields();
                   while (iter.hasNext())
                   {
-                    String fieldName = (String)iter.next();
-                    Object[] values = document.getField(fieldName);
+                    String fieldName = iter.next();
+                    String[] values = document.getFieldAsStrings(fieldName);
                     // We only handle strings right now!!!
                     int k = 0;
                     while (k < values.length)

Modified: incubator/lcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java?rev=1132386&r1=1132385&r2=1132386&view=diff
==============================================================================
--- incubator/lcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java
(original)
+++ incubator/lcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java
Sun Jun  5 12:36:30 2011
@@ -1116,12 +1116,12 @@ public class HttpPoster
 		  {
 		    if (newFieldName.toLowerCase().equals(idAttributeName.toLowerCase()))
 		      newFieldName = ID_METADATA;
-                    Object[] values = document.getField(fieldName);
+                    String[] values = document.getFieldAsStrings(fieldName);
                     // We only handle strings right now!!!
                     int k = 0;
                     while (k < values.length)
                     {
-                      String value = (String)values[k++];
+                      String value = values[k++];
                       totalLength += lengthField(LITERAL+newFieldName,value);
                     }
                   }
@@ -1180,12 +1180,12 @@ public class HttpPoster
 		  {
 		    if (newFieldName.toLowerCase().equals(idAttributeName.toLowerCase()))
 		      newFieldName = ID_METADATA;
-                    Object[] values = document.getField(fieldName);
+                    String[] values = document.getFieldAsStrings(fieldName);
                     // We only handle strings right now!!!
                     int k = 0;
                     while (k < values.length)
                     {
-                      String value = (String)values[k++];
+                      String value = values[k++];
                       writeField(out,LITERAL+newFieldName,value);
                     }
                   }

Modified: incubator/lcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/RepositoryDocument.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/RepositoryDocument.java?rev=1132386&r1=1132385&r2=1132386&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/RepositoryDocument.java
(original)
+++ incubator/lcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/RepositoryDocument.java
Sun Jun  5 12:36:30 2011
@@ -39,6 +39,8 @@ public class RepositoryDocument
   protected InputStream binaryFieldData = null;
   protected long binaryLength = 0;
   protected Map<String,Object> fields = new HashMap<String,Object>();
+  protected Map<String,String[]> stringFields = new HashMap<String,String[]>();
+  protected Map<String,Reader[]> readerFields = new HashMap<String,Reader[]>();
   protected Security fileSecurity = new Security();
   protected Security shareSecurity = new Security();
   protected List<Security> directorySecurity = new ArrayList<Security>();
@@ -174,9 +176,17 @@ public class RepositoryDocument
     throws ManifoldCFException
   {
     if (fieldData == null)
+    {
       fields.remove(fieldName);
+      stringFields.remove(fieldName);
+      readerFields.remove(fieldName);
+    }
     else
+    {
       fields.put(fieldName,fieldData);
+      stringFields.remove(fieldName);
+      readerFields.put(fieldName,fieldData);
+    }
   }
 
   /** Add a character field.
@@ -186,7 +196,7 @@ public class RepositoryDocument
   public void addField(String fieldName, Reader fieldData)
     throws ManifoldCFException
   {
-    fields.put(fieldName,new Reader[]{fieldData});
+    addField(fieldName,new Reader[]{fieldData});
   }
 
   /** Remove a multivalue character field.
@@ -198,9 +208,17 @@ public class RepositoryDocument
     throws ManifoldCFException
   {
     if (fieldData == null)
+    {
       fields.remove(fieldName);
+      stringFields.remove(fieldName);
+      readerFields.remove(fieldName);
+    }
     else
+    {
       fields.put(fieldName,fieldData);
+      readerFields.remove(fieldName);
+      stringFields.put(fieldName,fieldData);
+    }
   }
 
   /** Add a character field.
@@ -210,7 +228,7 @@ public class RepositoryDocument
   public void addField(String fieldName, String fieldData)
     throws ManifoldCFException
   {
-    fields.put(fieldName,new String[]{fieldData});
+    addField(fieldName,new String[]{fieldData});
   }
 
   /** Get a field.
@@ -222,6 +240,67 @@ public class RepositoryDocument
     return (Object[])fields.get(fieldName);
   }
 
+  /** Get a field as an array of strings.  If the data was originally in the form
+  * of Readers, a one-time conversion is made to the String form, so that the same
+  * field can be fetched multiple times.
+  *@param fieldName is the field name.
+  *@return the field data.
+  */
+  public String[] getFieldAsStrings(String fieldName)
+    throws IOException
+  {
+    String[] stringFieldData = stringFields.get(fieldName);
+    if (stringFieldData != null)
+      return stringFieldData;
+    Reader[] oldValues = readerFields.get(fieldName);
+    if (oldValues == null)
+      return null;
+    
+    String[] newValues = new String[oldValues.length];
+    char[] buffer = new char[65536];
+    int i = 0;
+    while (i < newValues.length)
+    {
+      Reader oldValue = oldValues[i];
+      StringBuilder newValue = new StringBuilder();
+      while (true)
+      {
+        int amt = oldValue.read(buffer);
+        if (amt == -1)
+          break;
+        newValue.append(buffer,0,amt);
+      }
+      newValues[i++] = newValue.toString();
+    }
+    stringFields.put(fieldName,newValues);
+    return newValues;
+  }
+
+  /** Get a field as an array of Readers.  If the field was originally
+  * strings, a one-time creation of a Readers array is made.
+  *@param fieldName is the field name.
+  *@return the field data.
+  */
+  public Reader[] getFieldAsReaders(String fieldName)
+  {
+    Reader[] readerFieldData = readerFields.get(fieldName);
+    if (readerFieldData != null)
+      return readerFieldData;
+    String[] oldValues = stringFields.get(fieldName);
+    if (oldValues == null)
+      return null;
+    
+    Reader[] newValues = new Reader[oldValues.length];
+    int i = 0;
+    while (i < newValues.length)
+    {
+      newValues[i] = new StringReader(oldValues[i]);
+      i++;
+    }
+    readerFields.put(fieldName,newValues);
+    return newValues;
+  }
+
   /** Get the number of fields.
   */
   public int fieldCount()



Mime
View raw message