manifoldcf-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kwri...@apache.org
Subject svn commit: r1493226 - in /manifoldcf/trunk: ./ connectors/googledrive/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/googledrive/ connectors/livelink/build-stub/src/main/java/com/opentext/api/ connectors/livelink/connector/src/main/j...
Date Fri, 14 Jun 2013 19:43:42 GMT
Author: kwright
Date: Fri Jun 14 19:43:42 2013
New Revision: 1493226

URL: http://svn.apache.org/r1493226
Log:
Fix for CONNECTORS-714. Revise LiveLink connector to allow document fetching via LAPI.

Added:
    manifoldcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/common/XThreadOutputStream.java
      - copied unchanged from r1493217, manifoldcf/branches/CONNECTORS-714/framework/core/src/main/java/org/apache/manifoldcf/core/common/XThreadOutputStream.java
Modified:
    manifoldcf/trunk/   (props changed)
    manifoldcf/trunk/CHANGES.txt
    manifoldcf/trunk/connectors/googledrive/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/googledrive/GoogleDriveSession.java
    manifoldcf/trunk/connectors/livelink/build-stub/src/main/java/com/opentext/api/LAPI_DOCUMENTS.java
    manifoldcf/trunk/connectors/livelink/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/livelink/LivelinkConnector.java
    manifoldcf/trunk/connectors/livelink/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/livelink/common_en_US.properties
    manifoldcf/trunk/connectors/livelink/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/livelink/common_ja_JP.properties

Propchange: manifoldcf/trunk/
------------------------------------------------------------------------------
  Merged /manifoldcf/branches/CONNECTORS-714:r1492346-1493217

Modified: manifoldcf/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/CHANGES.txt?rev=1493226&r1=1493225&r2=1493226&view=diff
==============================================================================
--- manifoldcf/trunk/CHANGES.txt (original)
+++ manifoldcf/trunk/CHANGES.txt Fri Jun 14 19:43:42 2013
@@ -3,6 +3,9 @@ $Id$
 
 ======================= 1.3-dev =====================
 
+CONNECTORS-714: All LiveLink connector to use LAPI for document fetches.
+(David Morana, Karl Wright)
+
 CONNECTORS-718: Alfresco Connector must throw exceptions with handler methods
 (Piergiorgio Lucidi, Karl Wright)
 

Modified: manifoldcf/trunk/connectors/googledrive/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/googledrive/GoogleDriveSession.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/googledrive/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/googledrive/GoogleDriveSession.java?rev=1493226&r1=1493225&r2=1493226&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/googledrive/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/googledrive/GoogleDriveSession.java (original)
+++ manifoldcf/trunk/connectors/googledrive/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/googledrive/GoogleDriveSession.java Fri Jun 14 19:43:42 2013
@@ -147,41 +147,4 @@ public class GoogleDriveSession {
     }
   }
   
-  protected static class XThreadOutputStream extends OutputStream {
-    protected final XThreadInputStream inputStream;
-    
-    byte[] byteBuffer = new byte[1];
-
-    public XThreadOutputStream(XThreadInputStream inputStream) {
-      this.inputStream = inputStream;
-    }
-    
-    @Override
-    public void write(int c)
-      throws IOException {
-      byteBuffer[0] = (byte)c;
-      try {
-        inputStream.stuffQueue(byteBuffer,0,1);
-      } catch (InterruptedException e) {
-        throw new InterruptedIOException(e.getMessage());
-      }
-    }
-
-    @Override
-    public void write(byte[] buffer, int pos, int amt)
-      throws IOException {
-      try {
-        inputStream.stuffQueue(buffer,pos,amt);
-      } catch (InterruptedException e) {
-        throw new InterruptedIOException(e.getMessage());
-      }
-    }
-    
-    @Override
-    public void close()
-      throws IOException {
-      inputStream.doneStuffingQueue();
-      super.close();
-    }
-  }
 }

Modified: manifoldcf/trunk/connectors/livelink/build-stub/src/main/java/com/opentext/api/LAPI_DOCUMENTS.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/livelink/build-stub/src/main/java/com/opentext/api/LAPI_DOCUMENTS.java?rev=1493226&r1=1493225&r2=1493226&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/livelink/build-stub/src/main/java/com/opentext/api/LAPI_DOCUMENTS.java (original)
+++ manifoldcf/trunk/connectors/livelink/build-stub/src/main/java/com/opentext/api/LAPI_DOCUMENTS.java Fri Jun 14 19:43:42 2013
@@ -53,6 +53,11 @@ public class LAPI_DOCUMENTS
     return 0;
   }
   
+  public int FetchVersion(int vol, int id, int revNumber, java.io.OutputStream output)
+  {
+    return 0;
+  }
+
   public int GetObjectRights(int vol, int objID, LLValue objinfo)
   {
     return 0;

Modified: manifoldcf/trunk/connectors/livelink/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/livelink/LivelinkConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/livelink/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/livelink/LivelinkConnector.java?rev=1493226&r1=1493225&r2=1493226&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/livelink/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/livelink/LivelinkConnector.java (original)
+++ manifoldcf/trunk/connectors/livelink/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/livelink/LivelinkConnector.java Fri Jun 14 19:43:42 2013
@@ -24,6 +24,7 @@ import org.apache.manifoldcf.crawler.int
 import org.apache.manifoldcf.crawler.system.Logging;
 import org.apache.manifoldcf.crawler.system.ManifoldCF;
 import org.apache.manifoldcf.core.common.XThreadInputStream;
+import org.apache.manifoldcf.core.common.XThreadOutputStream;
 
 import java.io.*;
 import java.util.*;
@@ -334,21 +335,31 @@ public class LivelinkConnector extends o
       serverHTTPNTLMPassword = params.getObfuscatedParameter(LiveLinkParameters.serverHTTPNTLMPassword);
 
       if (ingestProtocol == null || ingestProtocol.length() == 0)
-        ingestProtocol = "http";
+        ingestProtocol = null;
       if (viewProtocol == null || viewProtocol.length() == 0)
-        viewProtocol = ingestProtocol;
+      {
+        if (ingestProtocol == null)
+          viewProtocol = "http";
+        else
+          viewProtocol = ingestProtocol;
+      }
 
       if (ingestPort == null || ingestPort.length() == 0)
       {
-        if (!ingestProtocol.equals("https"))
-          ingestPort = "80";
+        if (ingestProtocol != null)
+        {
+          if (!ingestProtocol.equals("https"))
+            ingestPort = "80";
+          else
+            ingestPort = "443";
+        }
         else
-          ingestPort = "443";
+          ingestPort = null;
       }
 
       if (viewPort == null || viewPort.length() == 0)
       {
-        if (!viewProtocol.equals(ingestProtocol))
+        if (ingestProtocol == null || !viewProtocol.equals(ingestProtocol))
         {
           if (!viewProtocol.equals("https"))
             viewPort = "80";
@@ -359,13 +370,16 @@ public class LivelinkConnector extends o
           viewPort = ingestPort;
       }
 
-      try
+      if (ingestPort != null)
       {
-        ingestPortNumber = Integer.parseInt(ingestPort);
-      }
-      catch (NumberFormatException e)
-      {
-        throw new ManifoldCFException("Bad ingest port: "+e.getMessage(),e);
+        try
+        {
+          ingestPortNumber = Integer.parseInt(ingestPort);
+        }
+        catch (NumberFormatException e)
+        {
+          throw new ManifoldCFException("Bad ingest port: "+e.getMessage(),e);
+        }
       }
 
       String viewPortString;
@@ -580,62 +594,67 @@ public class LivelinkConnector extends o
       getSession();
 
       // Now, set up trial of ingestion connection
-      String contextMsg = "for document access";
-      String ingestHttpAddress = ingestCgiPath;
-
-      HttpClient client = getInitializedClient(contextMsg);
-      HttpGet method = new HttpGet(getHost().toURI() + ingestHttpAddress);
-      method.setHeader(new BasicHeader("Accept","*/*"));
-      try
+      if (ingestProtocol != null)
       {
-        int statusCode = executeMethodViaThread(client,method);
-        switch (statusCode)
+        String contextMsg = "for document access";
+        String ingestHttpAddress = ingestCgiPath;
+
+        HttpClient client = getInitializedClient(contextMsg);
+        HttpGet method = new HttpGet(getHost().toURI() + ingestHttpAddress);
+        method.setHeader(new BasicHeader("Accept","*/*"));
+        try
         {
-        case 502:
-          return "Fetch test had transient 502 error response";
+          int statusCode = executeMethodViaThread(client,method);
+          switch (statusCode)
+          {
+          case 502:
+            return "Fetch test had transient 502 error response";
 
-        case HttpStatus.SC_UNAUTHORIZED:
-          return "Fetch test returned UNAUTHORIZED (401) response; check the security credentials and configuration";
+          case HttpStatus.SC_UNAUTHORIZED:
+            return "Fetch test returned UNAUTHORIZED (401) response; check the security credentials and configuration";
 
-        case HttpStatus.SC_OK:
-          return super.check();
+          case HttpStatus.SC_OK:
+            return super.check();
 
-        default:
-          return "Fetch test returned an unexpected response code of "+Integer.toString(statusCode);
+          default:
+            return "Fetch test returned an unexpected response code of "+Integer.toString(statusCode);
+          }
+        }
+        catch (InterruptedException e)
+        {
+          throw new ManifoldCFException("Interrupted: "+e.getMessage(),e,ManifoldCFException.INTERRUPTED);
+        }
+        catch (java.net.SocketTimeoutException e)
+        {
+          return "Fetch test timed out reading from the Livelink HTTP Server: "+e.getMessage();
+        }
+        catch (java.net.SocketException e)
+        {
+          return "Fetch test received a socket error reading from Livelink HTTP Server: "+e.getMessage();
+        }
+        catch (javax.net.ssl.SSLHandshakeException e)
+        {
+          return "Fetch test was unable to set up a SSL connection to Livelink HTTP Server: "+e.getMessage();
+        }
+        catch (ConnectTimeoutException e)
+        {
+          return "Fetch test connection timed out reading from Livelink HTTP Server: "+e.getMessage();
+        }
+        catch (InterruptedIOException e)
+        {
+          throw new ManifoldCFException("Interrupted: "+e.getMessage(),e,ManifoldCFException.INTERRUPTED);
+        }
+        catch (HttpException e)
+        {
+          return "Fetch test had an HTTP exception: "+e.getMessage();
+        }
+        catch (IOException e)
+        {
+          return "Fetch test had an IO failure: "+e.getMessage();
         }
       }
-      catch (InterruptedException e)
-      {
-        throw new ManifoldCFException("Interrupted: "+e.getMessage(),e,ManifoldCFException.INTERRUPTED);
-      }
-      catch (java.net.SocketTimeoutException e)
-      {
-        return "Fetch test timed out reading from the Livelink HTTP Server: "+e.getMessage();
-      }
-      catch (java.net.SocketException e)
-      {
-        return "Fetch test received a socket error reading from Livelink HTTP Server: "+e.getMessage();
-      }
-      catch (javax.net.ssl.SSLHandshakeException e)
-      {
-        return "Fetch test was unable to set up a SSL connection to Livelink HTTP Server: "+e.getMessage();
-      }
-      catch (ConnectTimeoutException e)
-      {
-        return "Fetch test connection timed out reading from Livelink HTTP Server: "+e.getMessage();
-      }
-      catch (InterruptedIOException e)
-      {
-        throw new ManifoldCFException("Interrupted: "+e.getMessage(),e,ManifoldCFException.INTERRUPTED);
-      }
-      catch (HttpException e)
-      {
-        return "Fetch test had an HTTP exception: "+e.getMessage();
-      }
-      catch (IOException e)
-      {
-        return "Fetch test had an IO failure: "+e.getMessage();
-      }
+      else
+        return super.check();
     }
     catch (ServiceInterruption e)
     {
@@ -1686,16 +1705,23 @@ public class LivelinkConnector extends o
 "    editconnection.serverhttpcgipath.focus();\n"+
 "    return false;\n"+
 "  }\n"+
-"  if (editconnection.ingestcgipath.value == \"\")\n"+
+"  if (editconnection.viewprotocol.value == \"\" && editconnection.ingestprotocol.value == \"\")\n"+
 "  {\n"+
-"    alert(\""+Messages.getBodyJavascriptString(locale,"LivelinkConnector.EnterTheCrawlCgiPathToLivelink")+"\");\n"+
-"    SelectTab(\"" + Messages.getBodyJavascriptString(locale,"LivelinkConnector.DocumentAccess") + "\");\n"+
-"    editconnection.ingestcgipath.focus();\n"+
+"    alert(\""+Messages.getBodyJavascriptString(locale,"LivelinkConnector.SelectAViewProtocol")+"\");\n"+
+"    SelectTab(\"" + Messages.getBodyJavascriptString(locale,"LivelinkConnector.DocumentView") + "\");\n"+
+"    editconnection.viewprotocol.focus();\n"+
+"    return false;\n"+
+"  }\n"+
+"  if (editconnection.viewcgipath.value == \"\" && editconnection.ingestcgipath.value == \"\")\n"+
+"  {\n"+
+"    alert(\""+Messages.getBodyJavascriptString(locale,"LivelinkConnector.EnterTheViewCgiPathToLivelink")+"\");\n"+
+"    SelectTab(\"" + Messages.getBodyJavascriptString(locale,"LivelinkConnector.DocumentView") + "\");\n"+
+"    editconnection.viewcgipath.focus();\n"+
 "    return false;\n"+
 "  }\n"+
-"  if (editconnection.ingestcgipath.value.substring(0,1) != \"/\")\n"+
+"  if (editconnection.ingestcgipath.value != \"\" && editconnection.ingestcgipath.value.substring(0,1) != \"/\")\n"+
 "  {\n"+
-"    alert(\""+Messages.getBodyJavascriptString(locale,"LivelinkConnector.TheIngestCgiPathMustBeginWithACharacter")+"\");\n"+
+"    alert(\""+Messages.getBodyJavascriptString(locale,"LivelinkConnector.TheIngestCgiPathMustBeBlankOrBeginWithACharacter")+"\");\n"+
 "    SelectTab(\"" + Messages.getBodyJavascriptString(locale,"LivelinkConnector.DocumentAccess") + "\");\n"+
 "    editconnection.ingestcgipath.focus();\n"+
 "    return false;\n"+
@@ -1768,13 +1794,13 @@ public class LivelinkConnector extends o
     // Document access parameters
     String ingestProtocol = parameters.getParameter(LiveLinkParameters.ingestProtocol);
     if (ingestProtocol == null)
-      ingestProtocol = "http";
+      ingestProtocol = "";
     String ingestPort = parameters.getParameter(LiveLinkParameters.ingestPort);
     if (ingestPort == null)
       ingestPort = "";
     String ingestCgiPath = parameters.getParameter(LiveLinkParameters.ingestCgiPath);
     if (ingestCgiPath == null)
-      ingestCgiPath = "/livelink/livelink.exe";
+      ingestCgiPath = "";
     String ingestNtlmUsername = parameters.getParameter(LiveLinkParameters.ingestNtlmUsername);
     if (ingestNtlmUsername == null)
       ingestNtlmUsername = "";
@@ -1794,7 +1820,7 @@ public class LivelinkConnector extends o
     // Document view parameters
     String viewProtocol = parameters.getParameter(LiveLinkParameters.viewProtocol);
     if (viewProtocol == null)
-      viewProtocol = "";
+      viewProtocol = "http";
     String viewServerName = parameters.getParameter(LiveLinkParameters.viewServerName);
     if (viewServerName == null)
       viewServerName = "";
@@ -1803,7 +1829,7 @@ public class LivelinkConnector extends o
       viewPort = "";
     String viewCgiPath = parameters.getParameter(LiveLinkParameters.viewCgiPath);
     if (viewCgiPath == null)
-      viewCgiPath = "";
+      viewCgiPath = "/livelink/livelink.exe";
 
     // The "Server" tab
     // Always pass the whole keystore as a hidden.
@@ -1951,7 +1977,8 @@ public class LivelinkConnector extends o
 "  <tr>\n"+
 "    <td class=\"description\">"+Messages.getBodyString(locale,"LivelinkConnector.DocumentFetchProtocol")+"</td>\n"+
 "    <td class=\"value\">\n"+
-"      <select name=\"ingestprotocol\" size=\"2\">\n"+
+"      <select name=\"ingestprotocol\" size=\"3\">\n"+
+"        <option value=\"\" "+((ingestProtocol.equals(""))?"selected=\"selected\"":"")+">"+Messages.getBodyString(locale,"LivelinkConnector.UseLAPI")+"</option>\n"+
 "        <option value=\"http\" "+((ingestProtocol.equals("http"))?"selected=\"selected\"":"")+">http</option>\n"+
 "        <option value=\"https\" "+((ingestProtocol.equals("https"))?"selected=\"selected\"":"")+">https</option>\n"+
 "      </select>\n"+
@@ -4175,14 +4202,6 @@ public class LivelinkConnector extends o
 
     String contextMsg = "for '"+documentIdentifier+"'";
 
-    String ingestHttpAddress = convertToIngestURI(documentIdentifier);
-    if (ingestHttpAddress == null)
-    {
-      if (Logging.connectors.isDebugEnabled())
-        Logging.connectors.debug("Livelink: No fetch URI "+contextMsg+" - not ingesting");
-      return;
-    }
-
     String viewHttpAddress = convertToViewURI(documentIdentifier);
     if (viewHttpAddress == null)
     {
@@ -4191,13 +4210,17 @@ public class LivelinkConnector extends o
       return;
     }
 
-    RepositoryDocument rd = new RepositoryDocument();
-
-    int colonPos = documentIdentifier.indexOf(":",1);
-    
+    // Fetch logging
+    long startTime = System.currentTimeMillis();
+    String resultCode = "FAILED";
+    String resultDescription = null;
+    Long readSize = null;
+    boolean wasInterrupted = false;
     int objID;
     int vol;
 
+    int colonPos = documentIdentifier.indexOf(":",1);
+        
     if (colonPos == -1)
     {
       objID = new Integer(documentIdentifier.substring(1)).intValue();
@@ -4208,356 +4231,504 @@ public class LivelinkConnector extends o
       objID = new Integer(documentIdentifier.substring(colonPos+1)).intValue();
       vol = new Integer(documentIdentifier.substring(1,colonPos)).intValue();
     }
-
-    // Add general metadata
-    ObjectInformation objInfo = llc.getObjectInformation(vol,objID);
-    VersionInformation versInfo = llc.getVersionInformation(vol,objID,0);
-    if (!objInfo.exists())
-    {
-      Logging.connectors.debug("Livelink: No object "+contextMsg+": not ingesting");
-      return;
-    }
-    if (!versInfo.exists())
-    {
-      Logging.connectors.debug("Livelink: No version data "+contextMsg+": not ingesting");
-      return;
-    }
     
-    // Add general data we need for the output connector
-    String mimeType = versInfo.getMimeType();
-    if (mimeType != null)
-      rd.setMimeType(mimeType);
-    String fileName = versInfo.getFileName();
-    if (fileName != null)
-      rd.setFileName(fileName);
-    Date creationDate = objInfo.getCreationDate();
-    if (creationDate != null)
-      rd.setCreatedDate(creationDate);
-    Date modifyDate = versInfo.getModifyDate();
-    if (modifyDate != null)
-      rd.setModifiedDate(modifyDate);
-    
-    rd.addField(GENERAL_NAME_FIELD,objInfo.getName());
-    rd.addField(GENERAL_DESCRIPTION_FIELD,objInfo.getComments());
-    if (creationDate != null)
-      rd.addField(GENERAL_CREATIONDATE_FIELD,creationDate.toString());
-    if (modifyDate != null)
-      rd.addField(GENERAL_MODIFYDATE_FIELD,modifyDate.toString());
-    UserInformation owner = llc.getUserInformation(objInfo.getOwnerId().intValue());
-    UserInformation creator = llc.getUserInformation(objInfo.getCreatorId().intValue());
-    UserInformation modifier = llc.getUserInformation(versInfo.getOwnerId().intValue());
-    if (owner != null)
-      rd.addField(GENERAL_OWNER,owner.getName());
-    if (creator != null)
-      rd.addField(GENERAL_CREATOR,creator.getName());
-    if (modifier != null)
-      rd.addField(GENERAL_MODIFIER,modifier.getName());
-
-    // Iterate over the metadata items.  These are organized by category
-    // for speed of lookup.
-
-    // Unpack version string
-    int startPos = 0;
-
-    // Metadata items first
-    ArrayList metadataItems = new ArrayList();
-    startPos = unpackList(metadataItems,version,startPos,'+');
-    Iterator catIter = desc.getItems(metadataItems);
-    while (catIter.hasNext())
-    {
-      MetadataItem item = (MetadataItem)catIter.next();
-      MetadataPathItem pathItem = item.getPathItem();
-      if (pathItem != null)
-      {
-        int catID = pathItem.getCatID();
-        // grab the associated catversion
-        LLValue catVersion = getCatVersion(objID,catID);
-        if (catVersion != null)
-        {
-          // Go through attributes now
-          Iterator attrIter = item.getAttributeNames();
-          while (attrIter.hasNext())
-          {
-            String attrName = (String)attrIter.next();
-            // Create a unique metadata name
-            String metadataName = pathItem.getCatName()+":"+attrName;
-            // Fetch the metadata and stuff it into the RepositoryData structure
-            String[] metadataValue = getAttributeValue(catVersion,attrName);
-            if (metadataValue != null)
-              rd.addField(metadataName,metadataValue);
-            else
-              Logging.connectors.warn("Livelink: Metadata attribute '"+metadataName+"' does not seem to exist; please correct the job");
-          }
-        }
-
-      }
-    }
-
-    // Unpack acls (conditionally)
-    if (startPos < version.length())
+    // Try/finally for fetch logging
+    try
     {
-      char x = version.charAt(startPos++);
-      if (x == '+')
+      // Check URL first
+      if (activities.checkURLIndexable(viewHttpAddress))
       {
-        ArrayList acls = new ArrayList();
-        startPos = unpackList(acls,version,startPos,'+');
-        // Turn into acls and add into description
-        String[] aclArray = new String[acls.size()];
-        int j = 0;
-        while (j < aclArray.length)
+
+        // Add general metadata
+        ObjectInformation objInfo = llc.getObjectInformation(vol,objID);
+        VersionInformation versInfo = llc.getVersionInformation(vol,objID,0);
+        if (!objInfo.exists())
         {
-          aclArray[j] = (String)acls.get(j);
-          j++;
+          resultCode = "OBJECTNOTFOUND";
+          Logging.connectors.debug("Livelink: No object "+contextMsg+": not ingesting");
+          return;
+        }
+        if (!versInfo.exists())
+        {
+          resultCode = "VERSIONNOTFOUND";
+          Logging.connectors.debug("Livelink: No version data "+contextMsg+": not ingesting");
+          return;
         }
-        rd.setACL(aclArray);
 
-        StringBuilder denyBuffer = new StringBuilder();
-        startPos = unpack(denyBuffer,version,startPos,'+');
-        String denyAcl = denyBuffer.toString();
-        String[] denyAclArray = new String[1];
-        denyAclArray[0] = denyAcl;
-        rd.setDenyACL(denyAclArray);
-      }
-    }
+        String mimeType = versInfo.getMimeType();
+        if (activities.checkMimeTypeIndexable(mimeType))
+        {
+          Long dataSize = versInfo.getDataSize();
+          if (dataSize != null && activities.checkLengthIndexable(dataSize.longValue()))
+          {
+            String fileName = versInfo.getFileName();
+            Date creationDate = objInfo.getCreationDate();
+            Date modifyDate = versInfo.getModifyDate();
+            RepositoryDocument rd = new RepositoryDocument();
+
+            
+            // Add general data we need for the output connector
+            if (mimeType != null)
+              rd.setMimeType(mimeType);
+            if (fileName != null)
+              rd.setFileName(fileName);
+            if (creationDate != null)
+              rd.setCreatedDate(creationDate);
+            if (modifyDate != null)
+              rd.setModifiedDate(modifyDate);
+            
+            rd.addField(GENERAL_NAME_FIELD,objInfo.getName());
+            rd.addField(GENERAL_DESCRIPTION_FIELD,objInfo.getComments());
+            if (creationDate != null)
+              rd.addField(GENERAL_CREATIONDATE_FIELD,creationDate.toString());
+            if (modifyDate != null)
+              rd.addField(GENERAL_MODIFYDATE_FIELD,modifyDate.toString());
+            UserInformation owner = llc.getUserInformation(objInfo.getOwnerId().intValue());
+            UserInformation creator = llc.getUserInformation(objInfo.getCreatorId().intValue());
+            UserInformation modifier = llc.getUserInformation(versInfo.getOwnerId().intValue());
+            if (owner != null)
+              rd.addField(GENERAL_OWNER,owner.getName());
+            if (creator != null)
+              rd.addField(GENERAL_CREATOR,creator.getName());
+            if (modifier != null)
+              rd.addField(GENERAL_MODIFIER,modifier.getName());
+
+            // Iterate over the metadata items.  These are organized by category
+            // for speed of lookup.
+
+            // Unpack version string
+            int startPos = 0;
+
+            // Metadata items first
+            ArrayList metadataItems = new ArrayList();
+            startPos = unpackList(metadataItems,version,startPos,'+');
+            Iterator catIter = desc.getItems(metadataItems);
+            while (catIter.hasNext())
+            {
+              MetadataItem item = (MetadataItem)catIter.next();
+              MetadataPathItem pathItem = item.getPathItem();
+              if (pathItem != null)
+              {
+                int catID = pathItem.getCatID();
+                // grab the associated catversion
+                LLValue catVersion = getCatVersion(objID,catID);
+                if (catVersion != null)
+                {
+                  // Go through attributes now
+                  Iterator attrIter = item.getAttributeNames();
+                  while (attrIter.hasNext())
+                  {
+                    String attrName = (String)attrIter.next();
+                    // Create a unique metadata name
+                    String metadataName = pathItem.getCatName()+":"+attrName;
+                    // Fetch the metadata and stuff it into the RepositoryData structure
+                    String[] metadataValue = getAttributeValue(catVersion,attrName);
+                    if (metadataValue != null)
+                      rd.addField(metadataName,metadataValue);
+                    else
+                      Logging.connectors.warn("Livelink: Metadata attribute '"+metadataName+"' does not seem to exist; please correct the job");
+                  }
+                }
 
-    // Add the path metadata item into the mix, if enabled
-    String pathAttributeName = sDesc.getPathAttributeName();
-    if (pathAttributeName != null && pathAttributeName.length() > 0)
-    {
-      String pathString = sDesc.getPathAttributeValue(documentIdentifier);
-      if (pathString != null)
-      {
-        if (Logging.connectors.isDebugEnabled())
-          Logging.connectors.debug("Livelink: Path attribute name is '"+pathAttributeName+"'"+contextMsg+", value is '"+pathString+"'");
-        rd.addField(pathAttributeName,pathString);
-      }
-    }
+              }
+            }
 
-    // Set up connection
-    HttpClient client = getInitializedClient(contextMsg);
+            // Unpack acls (conditionally)
+            if (startPos < version.length())
+            {
+              char x = version.charAt(startPos++);
+              if (x == '+')
+              {
+                ArrayList acls = new ArrayList();
+                startPos = unpackList(acls,version,startPos,'+');
+                // Turn into acls and add into description
+                String[] aclArray = new String[acls.size()];
+                int j = 0;
+                while (j < aclArray.length)
+                {
+                  aclArray[j] = (String)acls.get(j);
+                  j++;
+                }
+                rd.setACL(aclArray);
 
-    long currentTime;
+                StringBuilder denyBuffer = new StringBuilder();
+                startPos = unpack(denyBuffer,version,startPos,'+');
+                String denyAcl = denyBuffer.toString();
+                String[] denyAclArray = new String[1];
+                denyAclArray[0] = denyAcl;
+                rd.setDenyACL(denyAclArray);
+              }
+            }
 
-    if (Logging.connectors.isInfoEnabled())
-      Logging.connectors.info("Livelink: " + ingestHttpAddress);
+            // Add the path metadata item into the mix, if enabled
+            String pathAttributeName = sDesc.getPathAttributeName();
+            if (pathAttributeName != null && pathAttributeName.length() > 0)
+            {
+              String pathString = sDesc.getPathAttributeValue(documentIdentifier);
+              if (pathString != null)
+              {
+                if (Logging.connectors.isDebugEnabled())
+                  Logging.connectors.debug("Livelink: Path attribute name is '"+pathAttributeName+"'"+contextMsg+", value is '"+pathString+"'");
+                rd.addField(pathAttributeName,pathString);
+              }
+            }
 
-    long startTime = System.currentTimeMillis();
-    String resultCode = "OK";
-    String resultDescription = null;
-    Long readSize = null;
+            if (viewProtocol != null)
+            {
+              // Use HTTP to fetch document!
+              String ingestHttpAddress = convertToIngestURI(documentIdentifier);
+              if (ingestHttpAddress != null)
+              {
 
-    HttpGet method = new HttpGet(getHost().toURI() + ingestHttpAddress);
-    method.setHeader(new BasicHeader("Accept","*/*"));
+                // Set up connection
+                HttpClient client = getInitializedClient(contextMsg);
 
-    ExecuteMethodThread methodThread = new ExecuteMethodThread(client,method);
-    methodThread.start();
-    try
-    {
+                long currentTime;
 
-      int statusCode = methodThread.getResponseCode();
-      switch (statusCode)
-      {
-      case 500:
-      case 502:
-        Logging.connectors.warn("Livelink: Service interruption during fetch "+contextMsg+" with Livelink HTTP Server, retrying...");
-        throw new ServiceInterruption("Service interruption during fetch",new ManifoldCFException(Integer.toString(statusCode)+" error while fetching"),System.currentTimeMillis()+60000L,
-          System.currentTimeMillis()+600000L,-1,true);
+                if (Logging.connectors.isInfoEnabled())
+                  Logging.connectors.info("Livelink: " + ingestHttpAddress);
 
-      case HttpStatus.SC_UNAUTHORIZED:
-        Logging.connectors.warn("Livelink: Document fetch unauthorized for "+ingestHttpAddress+" ("+contextMsg+")");
-        // Since we logged in, we should fail here if the ingestion user doesn't have access to the
-        // the document, but if we do, don't fail hard.
-        resultCode = "UNAUTHORIZED";
-        activities.deleteDocument(documentIdentifier,version);
-        return;
 
-      case HttpStatus.SC_OK:
-        if (Logging.connectors.isDebugEnabled())
-          Logging.connectors.debug("Livelink: Created http document connection to Livelink "+contextMsg);
-        long dataSize = methodThread.getResponseContentLength();
-        // The above replaces this, which required another access:
-        // long dataSize = (long)value.toInteger("DataSize");
-        // A non-existent content length will cause a value of -1 to be returned.  This seems to indicate that the session login did not work right.
-        if (dataSize >= 0)
-        {
-          if (Logging.connectors.isDebugEnabled())
-            Logging.connectors.debug("Livelink: Content length from livelink server "+contextMsg+"' = "+new Long(dataSize).toString());
-          if (activities.checkLengthIndexable(dataSize))
-          {
-            try
-            {
-              InputStream is = methodThread.getSafeInputStream();
-              try
-              {
-                rd.setBinary(is,dataSize);
-                
-                activities.ingestDocument(documentIdentifier,version,viewHttpAddress,rd);
+                HttpGet method = new HttpGet(getHost().toURI() + ingestHttpAddress);
+                method.setHeader(new BasicHeader("Accept","*/*"));
 
-                if (Logging.connectors.isDebugEnabled())
-                  Logging.connectors.debug("Livelink: Ingesting done "+contextMsg);
+                ExecuteMethodThread methodThread = new ExecuteMethodThread(client,method);
+                methodThread.start();
+                try
+                {
 
+                  int statusCode = methodThread.getResponseCode();
+                  switch (statusCode)
+                  {
+                  case 500:
+                  case 502:
+                    Logging.connectors.warn("Livelink: Service interruption during fetch "+contextMsg+" with Livelink HTTP Server, retrying...");
+                    throw new ServiceInterruption("Service interruption during fetch",new ManifoldCFException(Integer.toString(statusCode)+" error while fetching"),System.currentTimeMillis()+60000L,
+                      System.currentTimeMillis()+600000L,-1,true);
+
+                  case HttpStatus.SC_UNAUTHORIZED:
+                    Logging.connectors.warn("Livelink: Document fetch unauthorized for "+ingestHttpAddress+" ("+contextMsg+")");
+                    // Since we logged in, we should fail here if the ingestion user doesn't have access to the
+                    // the document, but if we do, don't fail hard.
+                    resultCode = "UNAUTHORIZED";
+                    activities.deleteDocument(documentIdentifier,version);
+                    return;
+
+                  case HttpStatus.SC_OK:
+                    if (Logging.connectors.isDebugEnabled())
+                      Logging.connectors.debug("Livelink: Created http document connection to Livelink "+contextMsg);
+                    // A non-existent content length will cause a value of -1 to be returned.  This seems to indicate that the session login did not work right.
+                    if (methodThread.getResponseContentLength() >= 0)
+                    {
+                      try
+                      {
+                        InputStream is = methodThread.getSafeInputStream();
+                        try
+                        {
+                          rd.setBinary(is,dataSize);
+                            
+                          activities.ingestDocument(documentIdentifier,version,viewHttpAddress,rd);
+
+                          if (Logging.connectors.isDebugEnabled())
+                            Logging.connectors.debug("Livelink: Ingesting done "+contextMsg);
+
+                        }
+                        finally
+                        {
+                          // Close stream via thread, since otherwise this can hang
+                          is.close();
+                        }
+                      }
+                      catch (java.net.SocketTimeoutException e)
+                      {
+                        resultCode = "DATATIMEOUT";
+                        resultDescription = e.getMessage();
+                        currentTime = System.currentTimeMillis();
+                        Logging.connectors.warn("Livelink: Livelink socket timed out ingesting from the Livelink HTTP Server "+contextMsg+": "+e.getMessage(), e);
+                        throw new ServiceInterruption("Socket timed out: "+e.getMessage(),e,currentTime+300000L,currentTime+6*3600000L,-1,false);
+                      }
+                      catch (java.net.SocketException e)
+                      {
+                        resultCode = "DATASOCKETERROR";
+                        resultDescription = e.getMessage();
+                        currentTime = System.currentTimeMillis();
+                        Logging.connectors.warn("Livelink: Livelink socket error ingesting from the Livelink HTTP Server "+contextMsg+": "+e.getMessage(), e);
+                        throw new ServiceInterruption("Socket error: "+e.getMessage(),e,currentTime+300000L,currentTime+6*3600000L,-1,false);
+                      }
+                      catch (javax.net.ssl.SSLHandshakeException e)
+                      {
+                        resultCode = "DATASSLHANDSHAKEERROR";
+                        resultDescription = e.getMessage();
+                        currentTime = System.currentTimeMillis();
+                        Logging.connectors.warn("Livelink: SSL handshake failed authenticating "+contextMsg+": "+e.getMessage(),e);
+                        throw new ServiceInterruption("SSL handshake error: "+e.getMessage(),e,currentTime+60000L,currentTime+300000L,-1,true);
+                      }
+                      catch (ConnectTimeoutException e)
+                      {
+                        resultCode = "CONNECTTIMEOUT";
+                        resultDescription = e.getMessage();
+                        currentTime = System.currentTimeMillis();
+                        Logging.connectors.warn("Livelink: Livelink socket timed out connecting to the Livelink HTTP Server "+contextMsg+": "+e.getMessage(), e);
+                        throw new ServiceInterruption("Connect timed out: "+e.getMessage(),e,currentTime+300000L,currentTime+6*3600000L,-1,false);
+                      }
+                      catch (InterruptedException e)
+                      {
+                        wasInterrupted = true;
+                        throw new ManifoldCFException("Interrupted: "+e.getMessage(),e,ManifoldCFException.INTERRUPTED);
+                      }
+                      catch (InterruptedIOException e)
+                      {
+                        wasInterrupted = true;
+                        throw new ManifoldCFException("Interrupted: "+e.getMessage(),e,ManifoldCFException.INTERRUPTED);
+                      }
+                      catch (HttpException e)
+                      {
+                        resultCode = "HTTPEXCEPTION";
+                        resultDescription = e.getMessage();
+                        // Treat unknown error ingesting data as a transient condition
+                        currentTime = System.currentTimeMillis();
+                        Logging.connectors.warn("Livelink: HTTP exception ingesting "+contextMsg+": "+e.getMessage(),e);
+                        throw new ServiceInterruption("HTTP exception ingesting "+contextMsg+": "+e.getMessage(),e,currentTime+300000L,currentTime+6*3600000L,-1,false);
+                      }
+                      catch (IOException e)
+                      {
+                        resultCode = "DATAEXCEPTION";
+                        resultDescription = e.getMessage();
+                        // Treat unknown error ingesting data as a transient condition
+                        currentTime = System.currentTimeMillis();
+                        Logging.connectors.warn("Livelink: IO exception ingesting "+contextMsg+": "+e.getMessage(),e);
+                        throw new ServiceInterruption("IO exception ingesting "+contextMsg+": "+e.getMessage(),e,currentTime+300000L,currentTime+6*3600000L,-1,false);
+                      }
+                      readSize = dataSize;
+                    }
+                    else
+                    {
+                      resultCode = "SESSIONLOGINFAILED";
+                      activities.deleteDocument(documentIdentifier,version);
+                    }
+                    break;
+                  case HttpStatus.SC_BAD_REQUEST:
+                  case HttpStatus.SC_USE_PROXY:
+                  case HttpStatus.SC_GONE:
+                    resultCode = "ERROR "+Integer.toString(statusCode);
+                    throw new ManifoldCFException("Unrecoverable request failure; error = "+Integer.toString(statusCode));
+                  default:
+                    resultCode = "UNKNOWN";
+                    Logging.connectors.warn("Livelink: Attempt to retrieve document from '"+ingestHttpAddress+"' received a response of "+Integer.toString(statusCode)+"; retrying in one minute");
+                    currentTime = System.currentTimeMillis();
+                    throw new ServiceInterruption("Fetch failed; retrying in 1 minute",new ManifoldCFException("Fetch failed with unknown code "+Integer.toString(statusCode)),
+                      currentTime+60000L,currentTime+600000L,-1,true);
+                  }
+                }
+                catch (InterruptedException e)
+                {
+                  // Drop the connection on the floor
+                  methodThread.interrupt();
+                  methodThread = null;
+                  throw new ManifoldCFException("Interrupted: "+e.getMessage(),e,ManifoldCFException.INTERRUPTED);
+                }
+                catch (java.net.SocketTimeoutException e)
+                {
+                  Logging.connectors.warn("Livelink: Socket timed out reading from the Livelink HTTP Server "+contextMsg+": "+e.getMessage(), e);
+                  resultCode = "TIMEOUT";
+                  resultDescription = e.getMessage();
+                  currentTime = System.currentTimeMillis();
+                  throw new ServiceInterruption("Socket timed out: "+e.getMessage(),e,currentTime+300000L,currentTime+6*3600000L,-1,true);
+                }
+                catch (java.net.SocketException e)
+                {
+                  Logging.connectors.warn("Livelink: Socket error reading from Livelink HTTP Server "+contextMsg+": "+e.getMessage(), e);
+                  resultCode = "SOCKETERROR";
+                  resultDescription = e.getMessage();
+                  currentTime = System.currentTimeMillis();
+                  throw new ServiceInterruption("Socket error: "+e.getMessage(),e,currentTime+300000L,currentTime+6*3600000L,-1,true);
+                }
+                catch (javax.net.ssl.SSLHandshakeException e)
+                {
+                  currentTime = System.currentTimeMillis();
+                  Logging.connectors.warn("Livelink: SSL handshake failed "+contextMsg+": "+e.getMessage(),e);
+                  resultCode = "SSLHANDSHAKEERROR";
+                  resultDescription = e.getMessage();
+                  throw new ServiceInterruption("SSL handshake error: "+e.getMessage(),e,currentTime+60000L,currentTime+300000L,-1,true);
+                }
+                catch (ConnectTimeoutException e)
+                {
+                  Logging.connectors.warn("Livelink: Connect timed out reading from the Livelink HTTP Server "+contextMsg+": "+e.getMessage(), e);
+                  resultCode = "CONNECTTIMEOUT";
+                  resultDescription = e.getMessage();
+                  currentTime = System.currentTimeMillis();
+                  throw new ServiceInterruption("Connect timed out: "+e.getMessage(),e,currentTime+300000L,currentTime+6*3600000L,-1,true);
+                }
+                catch (InterruptedIOException e)
+                {
+                  methodThread.interrupt();
+                  throw new ManifoldCFException("Interrupted: "+e.getMessage(),e,ManifoldCFException.INTERRUPTED);
+                }
+                catch (HttpException e)
+                {
+                  resultCode = "EXCEPTION";
+                  resultDescription = e.getMessage();
+                  throw new ManifoldCFException("Exception getting response "+contextMsg+": "+e.getMessage(), e);
+                }
+                catch (IOException e)
+                {
+                  resultCode = "EXCEPTION";
+                  resultDescription = e.getMessage();
+                  throw new ManifoldCFException("Exception getting response "+contextMsg+": "+e.getMessage(), e);
+                }
+                finally
+                {
+                  if (methodThread != null)
+                  {
+                    methodThread.abort();
+                    if (!wasInterrupted)
+                    {
+                      try
+                      {
+                       methodThread.finishUp();
+                      }
+                      catch (InterruptedException e)
+                      {
+                        wasInterrupted = true;
+                        throw new ManifoldCFException(e.getMessage(),e,ManifoldCFException.INTERRUPTED);
+                      }
+                    }
+                  }
+                }
               }
-              finally
+              else
               {
-                // Close stream via thread, since otherwise this can hang
-                is.close();
+                if (Logging.connectors.isDebugEnabled())
+                  Logging.connectors.debug("Livelink: No fetch URI "+contextMsg+" - not ingesting");
+                resultCode = "NOURI";
+                return;
               }
             }
-            catch (java.net.SocketTimeoutException e)
-            {
-              resultCode = "DATATIMEOUT";
-              resultDescription = e.getMessage();
-              currentTime = System.currentTimeMillis();
-              Logging.connectors.warn("Livelink: Livelink socket timed out ingesting from the Livelink HTTP Server "+contextMsg+": "+e.getMessage(), e);
-              throw new ServiceInterruption("Socket timed out: "+e.getMessage(),e,currentTime+300000L,currentTime+6*3600000L,-1,false);
-            }
-            catch (java.net.SocketException e)
-            {
-              resultCode = "DATASOCKETERROR";
-              resultDescription = e.getMessage();
-              currentTime = System.currentTimeMillis();
-              Logging.connectors.warn("Livelink: Livelink socket error ingesting from the Livelink HTTP Server "+contextMsg+": "+e.getMessage(), e);
-              throw new ServiceInterruption("Socket error: "+e.getMessage(),e,currentTime+300000L,currentTime+6*3600000L,-1,false);
-            }
-            catch (javax.net.ssl.SSLHandshakeException e)
-            {
-              resultCode = "DATASSLHANDSHAKEERROR";
-              resultDescription = e.getMessage();
-              currentTime = System.currentTimeMillis();
-              Logging.connectors.warn("Livelink: SSL handshake failed authenticating "+contextMsg+": "+e.getMessage(),e);
-              throw new ServiceInterruption("SSL handshake error: "+e.getMessage(),e,currentTime+60000L,currentTime+300000L,-1,true);
-            }
-            catch (ConnectTimeoutException e)
-            {
-              resultCode = "CONNECTTIMEOUT";
-              resultDescription = e.getMessage();
-              currentTime = System.currentTimeMillis();
-              Logging.connectors.warn("Livelink: Livelink socket timed out connecting to the Livelink HTTP Server "+contextMsg+": "+e.getMessage(), e);
-              throw new ServiceInterruption("Connect timed out: "+e.getMessage(),e,currentTime+300000L,currentTime+6*3600000L,-1,false);
-            }
-            catch (InterruptedException e)
-            {
-              throw new ManifoldCFException("Interrupted: "+e.getMessage(),e,ManifoldCFException.INTERRUPTED);
-            }
-            catch (InterruptedIOException e)
-            {
-              throw new ManifoldCFException("Interrupted: "+e.getMessage(),e,ManifoldCFException.INTERRUPTED);
-            }
-            catch (HttpException e)
-            {
-              resultCode = "HTTPEXCEPTION";
-              resultDescription = e.getMessage();
-              // Treat unknown error ingesting data as a transient condition
-              currentTime = System.currentTimeMillis();
-              Logging.connectors.warn("Livelink: HTTP exception ingesting "+contextMsg+": "+e.getMessage(),e);
-              throw new ServiceInterruption("HTTP exception ingesting "+contextMsg+": "+e.getMessage(),e,currentTime+300000L,currentTime+6*3600000L,-1,false);
-            }
-            catch (IOException e)
+            else
             {
-              resultCode = "DATAEXCEPTION";
-              resultDescription = e.getMessage();
-              // Treat unknown error ingesting data as a transient condition
-              currentTime = System.currentTimeMillis();
-              Logging.connectors.warn("Livelink: IO exception ingesting "+contextMsg+": "+e.getMessage(),e);
-              throw new ServiceInterruption("IO exception ingesting "+contextMsg+": "+e.getMessage(),e,currentTime+300000L,currentTime+6*3600000L,-1,false);
+              // Use FetchVersion instead
+              long currentTime;
+              
+              // Fire up the document reading thread
+              DocumentReadingThread t = new DocumentReadingThread(vol,objID,0);
+              try 
+              {
+                t.start();
+                try
+                {
+                  InputStream is = t.getSafeInputStream();
+                  try 
+                  {
+                    // Can only index while background thread is running!
+                    rd.setBinary(is, dataSize);
+                    activities.ingestDocument(documentIdentifier, version, viewHttpAddress, rd);
+                  }
+                  finally
+                  {
+                    is.close();
+                  }
+                }
+                catch (ManifoldCFException e)
+                {
+                  if (e.getErrorCode() == ManifoldCFException.INTERRUPTED)
+                    wasInterrupted = true;
+                  throw e;
+                }
+                catch (java.net.SocketTimeoutException e)
+                {
+                  throw e;
+                }
+                catch (InterruptedIOException e)
+                {
+                  wasInterrupted = true;
+                  throw e;
+                }
+                finally
+                {
+                  if (!wasInterrupted)
+                    t.finishUp();
+                }
+
+                // No errors.  Record the fact that we made it.
+                resultCode = "OK";
+                readSize = dataSize;
+              }
+              catch (InterruptedException e) 
+              {
+                t.interrupt();
+                throw new ManifoldCFException("Interrupted: " + e.getMessage(), e,
+                  ManifoldCFException.INTERRUPTED);
+              }
+              catch (ConnectTimeoutException e)
+              {
+                Logging.connectors.warn("Livelink: Connect timed out "+contextMsg+": "+e.getMessage(), e);
+                resultCode = "CONNECTTIMEOUT";
+                resultDescription = e.getMessage();
+                currentTime = System.currentTimeMillis();
+                throw new ServiceInterruption("Connect timed out: "+e.getMessage(),e,currentTime+300000L,currentTime+6*3600000L,-1,true);
+              }
+              catch (InterruptedIOException e)
+              {
+                t.interrupt();
+                throw new ManifoldCFException("Interrupted: "+e.getMessage(),e,ManifoldCFException.INTERRUPTED);
+              }
+              catch (IOException e)
+              {
+                resultCode = "EXCEPTION";
+                resultDescription = e.getMessage();
+                throw new ManifoldCFException("Exception getting response "+contextMsg+": "+e.getMessage(), e);
+              }
+              catch (ManifoldCFException e)
+              {
+                if (e.getErrorCode() != ManifoldCFException.INTERRUPTED)
+                {
+                  resultCode = "EXCEPTION";
+                  resultDescription = e.getMessage();
+                }
+                throw e;
+              }
+              catch (RuntimeException e)
+              {
+                resultCode = "EXCEPTION";
+                resultDescription = e.getMessage();
+                handleLivelinkRuntimeException(e,0,true);
+              }
             }
-            readSize = new Long(dataSize);
           }
           else
           {
+            // Document not indexable because of its length
+            resultDescription = "Document length ("+dataSize+") was rejected by output connector";
+            if (Logging.connectors.isDebugEnabled())
+              Logging.connectors.debug("Livelink: Excluding document "+documentIdentifier+" because its length ("+dataSize+") was rejected by output connector");
             resultCode = "DOCUMENTTOOLONG";
             activities.deleteDocument(documentIdentifier,version);
           }
         }
         else
         {
-          resultCode = "SESSIONLOGINFAILED";
+          // Document not indexable because of its mime type
+          resultDescription = "Mime type ("+mimeType+") was rejected by output connector";
+          if (Logging.connectors.isDebugEnabled())
+            Logging.connectors.debug("Livelink: Excluding document "+documentIdentifier+" because its mime type ("+mimeType+") was rejected by output connector");
+          resultCode = "MIMETYPEEXCLUSION";
           activities.deleteDocument(documentIdentifier,version);
         }
-        break;
-      case HttpStatus.SC_BAD_REQUEST:
-      case HttpStatus.SC_USE_PROXY:
-      case HttpStatus.SC_GONE:
-        resultCode = "ERROR "+Integer.toString(statusCode);
-        throw new ManifoldCFException("Unrecoverable request failure; error = "+Integer.toString(statusCode));
-      default:
-        resultCode = "UNKNOWN";
-        Logging.connectors.warn("Livelink: Attempt to retrieve document from '"+ingestHttpAddress+"' received a response of "+Integer.toString(statusCode)+"; retrying in one minute");
-        currentTime = System.currentTimeMillis();
-        throw new ServiceInterruption("Fetch failed; retrying in 1 minute",new ManifoldCFException("Fetch failed with unknown code "+Integer.toString(statusCode)),
-          currentTime+60000L,currentTime+600000L,-1,true);
       }
-    }
-    catch (InterruptedException e)
-    {
-      // Drop the connection on the floor
-      methodThread.interrupt();
-      methodThread = null;
-      throw new ManifoldCFException("Interrupted: "+e.getMessage(),e,ManifoldCFException.INTERRUPTED);
-    }
-    catch (java.net.SocketTimeoutException e)
-    {
-      Logging.connectors.warn("Livelink: Socket timed out reading from the Livelink HTTP Server "+contextMsg+": "+e.getMessage(), e);
-      resultCode = "TIMEOUT";
-      resultDescription = e.getMessage();
-      currentTime = System.currentTimeMillis();
-      throw new ServiceInterruption("Socket timed out: "+e.getMessage(),e,currentTime+300000L,currentTime+6*3600000L,-1,true);
-    }
-    catch (java.net.SocketException e)
-    {
-      Logging.connectors.warn("Livelink: Socket error reading from Livelink HTTP Server "+contextMsg+": "+e.getMessage(), e);
-      resultCode = "SOCKETERROR";
-      resultDescription = e.getMessage();
-      currentTime = System.currentTimeMillis();
-      throw new ServiceInterruption("Socket error: "+e.getMessage(),e,currentTime+300000L,currentTime+6*3600000L,-1,true);
-    }
-    catch (javax.net.ssl.SSLHandshakeException e)
-    {
-      currentTime = System.currentTimeMillis();
-      Logging.connectors.warn("Livelink: SSL handshake failed "+contextMsg+": "+e.getMessage(),e);
-      resultCode = "SSLHANDSHAKEERROR";
-      resultDescription = e.getMessage();
-      throw new ServiceInterruption("SSL handshake error: "+e.getMessage(),e,currentTime+60000L,currentTime+300000L,-1,true);
-    }
-    catch (ConnectTimeoutException e)
-    {
-      Logging.connectors.warn("Livelink: Connect timed out reading from the Livelink HTTP Server "+contextMsg+": "+e.getMessage(), e);
-      resultCode = "CONNECTTIMEOUT";
-      resultDescription = e.getMessage();
-      currentTime = System.currentTimeMillis();
-      throw new ServiceInterruption("Connect timed out: "+e.getMessage(),e,currentTime+300000L,currentTime+6*3600000L,-1,true);
-    }
-    catch (InterruptedIOException e)
-    {
-      throw new ManifoldCFException("Interrupted: "+e.getMessage(),e,ManifoldCFException.INTERRUPTED);
-    }
-    catch (HttpException e)
-    {
-      resultCode = "EXCEPTION";
-      resultDescription = e.getMessage();
-      throw new ManifoldCFException("Exception getting response "+contextMsg+": "+e.getMessage(), e);
-    }
-    catch (IOException e)
-    {
-      resultCode = "EXCEPTION";
-      resultDescription = e.getMessage();
-      throw new ManifoldCFException("Exception getting response "+contextMsg+": "+e.getMessage(), e);
+      else
+      {
+        // Document not ingestable due to URL
+        resultDescription = "URL ("+viewHttpAddress+") was rejected by output connector";
+        if (Logging.connectors.isDebugEnabled())
+          Logging.connectors.debug("Livelink: Excluding document "+documentIdentifier+" because its URL ("+viewHttpAddress+") was rejected by output connector");
+        resultCode = "URLEXCLUSION";
+        activities.deleteDocument(documentIdentifier,version);
+      }
     }
     finally
     {
-      if (methodThread != null)
-      {
-        methodThread.abort();
-        activities.recordActivity(new Long(startTime),ACTIVITY_FETCH,readSize,Integer.toString(objID),resultCode,resultDescription,null);
-        try
-        {
-         methodThread.finishUp();
-        }
-        catch (InterruptedException e)
-        {
-          throw new ManifoldCFException(e.getMessage(),e,ManifoldCFException.INTERRUPTED);
-        }
-      }
+      if (!wasInterrupted)
+        activities.recordActivity(new Long(startTime),ACTIVITY_FETCH,readSize,vol+":"+objID,resultCode,resultDescription,null);
     }
   }
 
@@ -5430,7 +5601,18 @@ public class LivelinkConnector extends o
     {
       return getVersionValue() != null;
     }
-    
+
+    /** Get data size.
+    */
+    public Long getDataSize()
+      throws ServiceInterruption, ManifoldCFException
+    {
+      LLValue elem = getVersionValue();
+      if (elem == null)
+        return null;
+      return new Long(elem.toInteger("FILEDATASIZE"));
+    }
+
     /** Get file name.
     */
     public String getFileName()
@@ -5587,7 +5769,6 @@ public class LivelinkConnector extends o
       return "(Volume: "+volumeID+", Object: "+objectID+")";
     }
     
-
     /**
     * Returns the object ID specified by the path name.
     * @param startPath is the folder name (a string with dots as separators)
@@ -7275,6 +7456,79 @@ public class LivelinkConnector extends o
 
   }
 
+  /** This thread performs a LAPI FetchVersion command, streaming the resulting
+  * document back through a XThreadInputStream to the invoking thread.
+  */
+  protected class DocumentReadingThread extends Thread 
+  {
+
+    protected Throwable exception = null;
+    protected final int volumeID;
+    protected final int docID;
+    protected final int versionNumber;
+    protected final XThreadInputStream stream;
+    
+    public DocumentReadingThread(int volumeID, int docID, int versionNumber)
+    {
+      super();
+      this.volumeID = volumeID;
+      this.docID = docID;
+      this.versionNumber = versionNumber;
+      this.stream = new XThreadInputStream();
+      setDaemon(true);
+    }
+
+    @Override
+    public void run()
+    {
+      try
+      {
+        XThreadOutputStream outputStream = new XThreadOutputStream(stream);
+        try 
+        {
+          int status = LLDocs.FetchVersion(volumeID, docID, versionNumber, outputStream);
+          if (status != 0)
+          {
+            throw new ManifoldCFException("Error retrieving contents of document "+Integer.toString(volumeID)+":"+Integer.toString(docID)+" revision "+versionNumber+" : Status="+Integer.toString(status)+" ("+llServer.getErrors()+")");
+          }
+        }
+        finally
+        {
+          outputStream.close();
+        }
+      } catch (Throwable e) {
+        this.exception = e;
+      }
+    }
+
+    public InputStream getSafeInputStream() {
+      return stream;
+    }
+    
+    public void finishUp()
+      throws InterruptedException, ManifoldCFException
+    {
+      // This will be called during the finally
+      // block in the case where all is well (and
+      // the stream completed) and in the case where
+      // there were exceptions.
+      stream.abort();
+      join();
+      Throwable thr = exception;
+      if (thr != null) {
+        if (thr instanceof ManifoldCFException)
+          throw (ManifoldCFException) thr;
+        else if (thr instanceof RuntimeException)
+          throw (RuntimeException) thr;
+        else if (thr instanceof Error)
+          throw (Error) thr;
+        else
+          throw new RuntimeException("Unhandled exception of type: "+thr.getClass().getName(),thr);
+      }
+    }
+
+  }
+
   /** This thread does the actual socket communication with the server.
   * It's set up so that it can be abandoned at shutdown time.
   *

Modified: manifoldcf/trunk/connectors/livelink/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/livelink/common_en_US.properties
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/livelink/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/livelink/common_en_US.properties?rev=1493226&r1=1493225&r2=1493226&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/livelink/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/livelink/common_en_US.properties (original)
+++ manifoldcf/trunk/connectors/livelink/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/livelink/common_en_US.properties Fri Jun 14 19:43:42 2013
@@ -144,3 +144,8 @@ LivelinkConnector.TheServerCgiPathMustBe
 LivelinkConnector.Delete=Delete
 LivelinkConnector.Add=Add
 LivelinkConnector.CrawlUserWorkspaces=Crawl user workspaces?
+
+LivelinkConnector.EnterTheViewCgiPathToLivelink=Enter the view CGI path to LiveLink
+LivelinkConnector.TheIngestCgiPathMustBeBlankOrBeginWithACharacter=The ingestion CGI path must be blank or begin with a '/' character
+LivelinkConnector.UseLAPI=Use LAPI
+LivelinkConnector.SelectAViewProtocol=Select a view protocol

Modified: manifoldcf/trunk/connectors/livelink/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/livelink/common_ja_JP.properties
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/livelink/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/livelink/common_ja_JP.properties?rev=1493226&r1=1493225&r2=1493226&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/livelink/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/livelink/common_ja_JP.properties (original)
+++ manifoldcf/trunk/connectors/livelink/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/livelink/common_ja_JP.properties Fri Jun 14 19:43:42 2013
@@ -144,3 +144,8 @@ LivelinkConnector.TheServerCgiPathMustBe
 LivelinkConnector.Delete=削除
 LivelinkConnector.Add=追加
 LivelinkConnector.CrawlUserWorkspaces=Crawl user workspaces?
+
+LivelinkConnector.EnterTheViewCgiPathToLivelink=Enter the view CGI path to LiveLink
+LivelinkConnector.TheIngestCgiPathMustBeBlankOrBeginWithACharacter=The ingestion CGI path must be blank or begin with a '/' character
+LivelinkConnector.UseLAPI=Use LAPI
+LivelinkConnector.SelectAViewProtocol=Select a view protocol



Mime
View raw message