manifoldcf-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kwri...@apache.org
Subject svn commit: r1633636 - in /manifoldcf/branches/dev_1x: ./ connectors/livelink/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/livelink/LivelinkConnector.java
Date Wed, 22 Oct 2014 15:12:25 GMT
Author: kwright
Date: Wed Oct 22 15:12:25 2014
New Revision: 1633636

URL: http://svn.apache.org/r1633636
Log:
Pull up more fixes for CONNECTORS-1077 from trunk

Modified:
    manifoldcf/branches/dev_1x/   (props changed)
    manifoldcf/branches/dev_1x/connectors/livelink/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/livelink/LivelinkConnector.java

Propchange: manifoldcf/branches/dev_1x/
------------------------------------------------------------------------------
  Merged /manifoldcf/trunk:r1633634

Modified: manifoldcf/branches/dev_1x/connectors/livelink/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/livelink/LivelinkConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/connectors/livelink/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/livelink/LivelinkConnector.java?rev=1633636&r1=1633635&r2=1633636&view=diff
==============================================================================
--- manifoldcf/branches/dev_1x/connectors/livelink/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/livelink/LivelinkConnector.java (original)
+++ manifoldcf/branches/dev_1x/connectors/livelink/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/livelink/LivelinkConnector.java Wed Oct 22 15:12:25 2014
@@ -1175,264 +1175,258 @@ public class LivelinkConnector extends o
 
       getSession();
       ObjectInformation value = llc.getObjectInformation(vol,objID);
-      if (value.exists())
+      if (!value.exists())
       {
-        // Make sure we have permission to see the object's contents
-        int permissions = value.getPermissions().intValue();
-        if ((permissions & LAPI_DOCUMENTS.PERM_SEECONTENTS) != 0)
+        if (Logging.connectors.isDebugEnabled())
+          Logging.connectors.debug("Livelink: Object "+Integer.toString(vol)+":"+Integer.toString(objID)+" has no information - deleting");
+        activities.deleteDocument(documentIdentifier);
+        continue;
+      }
+        
+      // Make sure we have permission to see the object's contents
+      int permissions = value.getPermissions().intValue();
+      if ((permissions & LAPI_DOCUMENTS.PERM_SEECONTENTS) == 0)
+      {
+        if (Logging.connectors.isDebugEnabled())
+          Logging.connectors.debug("Livelink: Crawl user cannot see contents of object "+Integer.toString(vol)+":"+Integer.toString(objID)+" - deleting");
+        activities.deleteDocument(documentIdentifier);
+        continue;
+      }
+        
+        
+      Date dt = value.getModifyDate();
+      // The rights don't change when the object changes, so we have to include those too.
+      int[] rights = getObjectRights(vol,objID);
+      if (rights == null)
+      {
+        if (Logging.connectors.isDebugEnabled())
+          Logging.connectors.debug("Livelink: Could not get rights for object "+Integer.toString(vol)+":"+Integer.toString(objID)+" - deleting");
+        activities.deleteDocument(documentIdentifier);
+        continue;
+      }
+        
+      // We were able to get rights, so object still exists.
+          
+      // Changed folder versioning for MCF 2.0
+      if (isFolder)
+      {
+        // === Livelink folder ===
+        // I'm still not sure if Livelink folder modified dates are one-level or hierarchical.
+        // The code below assumes one-level only, so we always scan folders and there's no versioning
+        if (Logging.connectors.isDebugEnabled())
+          Logging.connectors.debug("Livelink: Processing folder "+Integer.toString(vol)+":"+Integer.toString(objID));
+
+        int sanityRetryCount = FAILURE_RETRY_COUNT;
+        while (true)
         {
-          Date dt = value.getModifyDate();
-          // The rights don't change when the object changes, so we have to include those too.
-          int[] rights = getObjectRights(vol,objID);
-          if (rights != null)
+          ListObjectsThread t = new ListObjectsThread(vol,objID,filterString);
+          try
           {
-            // We were able to get rights, so object still exists.
-
-            // Changed folder versioning for MCF 2.0
-            if (isFolder)
+            t.start();
+            LLValue childrenDocs;
+            try
             {
-              // === Livelink folder ===
-              // I'm still not sure if Livelink folder modified dates are one-level or hierarchical.
-              // The code below assumes one-level only, so we always scan folders and there's no versioning
-              if (Logging.connectors.isDebugEnabled())
-                Logging.connectors.debug("Livelink: Processing folder "+Integer.toString(vol)+":"+Integer.toString(objID));
-
-              int sanityRetryCount = FAILURE_RETRY_COUNT;
-              while (true)
-              {
-                ListObjectsThread t = new ListObjectsThread(vol,objID,filterString);
-                try
-                {
-                  t.start();
-                  LLValue childrenDocs;
-                  try
-                  {
-                    childrenDocs = t.finishUp();
-                  }
-                  catch (ManifoldCFException e)
-                  {
-                    sanityRetryCount = assessRetry(sanityRetryCount,e);
-                    continue;
-                  }
-
-                  int size = 0;
-                  
-                  if (childrenDocs.isRecord())
-                    size = 1;
-                  if (childrenDocs.isTable())
-                    size = childrenDocs.size();
-
-                  // System.out.println("Total child count = "+Integer.toString(size));
-                  // Do the scan
-                  for (int j = 0; j < size; j++)
-                  {
-                    int childID = childrenDocs.toInteger(j, "ID");
-
-                    if (Logging.connectors.isDebugEnabled())
-                      Logging.connectors.debug("Livelink: Found a child of folder "+Integer.toString(vol)+":"+Integer.toString(objID)+" : ID="+Integer.toString(childID));
-
-                    int subtype = childrenDocs.toInteger(j, "SubType");
-                    boolean childIsFolder = (subtype == LAPI_DOCUMENTS.FOLDERSUBTYPE || subtype == LAPI_DOCUMENTS.PROJECTSUBTYPE ||
-                      subtype == LAPI_DOCUMENTS.COMPOUNDDOCUMENTSUBTYPE);
-
-                    // If it's a folder, we just let it through for now
-                    if (!childIsFolder && checkInclude(childrenDocs.toString(j,"Name") + "." + childrenDocs.toString(j,"FileType"), spec) == false)
-                    {
-                      if (Logging.connectors.isDebugEnabled())
-                        Logging.connectors.debug("Livelink: Child identifier "+Integer.toString(childID)+" was excluded by inclusion criteria");
-                      continue;
-                    }
-
-                    if (childIsFolder)
-                    {
-                      if (Logging.connectors.isDebugEnabled())
-                        Logging.connectors.debug("Livelink: Child identifier "+Integer.toString(childID)+" is a folder, project, or compound document; adding a reference");
-                      if (subtype == LAPI_DOCUMENTS.PROJECTSUBTYPE)
-                      {
-                        // If we pick up a project object, we need to describe the volume object (which
-                        // will be the root of all documents beneath)
-                        activities.addDocumentReference("F"+new Integer(childID).toString()+":"+new Integer(-childID).toString());
-                      }
-                      else
-                        activities.addDocumentReference("F"+new Integer(vol).toString()+":"+new Integer(childID).toString());
-                    }
-                    else
-                    {
-                      if (Logging.connectors.isDebugEnabled())
-                        Logging.connectors.debug("Livelink: Child identifier "+Integer.toString(childID)+" is a simple document; adding a reference");
+              childrenDocs = t.finishUp();
+            }
+            catch (ManifoldCFException e)
+            {
+              sanityRetryCount = assessRetry(sanityRetryCount,e);
+              continue;
+            }
 
-                      activities.addDocumentReference("D"+new Integer(vol).toString()+":"+new Integer(childID).toString());
-                    }
+            int size = 0;
+            
+            if (childrenDocs.isRecord())
+              size = 1;
+            if (childrenDocs.isTable())
+              size = childrenDocs.size();
+
+            // System.out.println("Total child count = "+Integer.toString(size));
+            // Do the scan
+            for (int j = 0; j < size; j++)
+            {
+              int childID = childrenDocs.toInteger(j, "ID");
 
-                  }
-                  break;
-                }
-                catch (InterruptedException e)
-                {
-                  t.interrupt();
-                  throw new ManifoldCFException("Interrupted: "+e.getMessage(),e,ManifoldCFException.INTERRUPTED);
-                }
-                catch (RuntimeException e)
-                {
-                  sanityRetryCount = handleLivelinkRuntimeException(e,sanityRetryCount,true);
-                  continue;
-                }
-              }
               if (Logging.connectors.isDebugEnabled())
-                Logging.connectors.debug("Livelink: Done processing folder "+Integer.toString(vol)+":"+Integer.toString(objID));
-            }
-            else
-            {
-              // === Livelink document ===
-              
-              // The version string includes the following:
-              // 1) The modify date for the document
-              // 2) The rights for the document, ordered (which can change without changing the ModifyDate field)
-              // 3) The requested metadata fields (category and attribute, ordered) for the document
-              //
-              // The document identifiers are object id's.
-      
-              StringBuilder sb = new StringBuilder();
+                Logging.connectors.debug("Livelink: Found a child of folder "+Integer.toString(vol)+":"+Integer.toString(objID)+" : ID="+Integer.toString(childID));
 
-              String[] categoryPaths;
-              if (sDesc.includeAllMetadata())
-              {
-                // Find all the metadata associated with this object, and then
-                // find the set of category pathnames that correspond to it.
-                int[] catIDs = getObjectCategoryIDs(vol,objID);
-                categoryPaths = catAccum.getCategoryPathsAttributeNames(catIDs);
-                // Sort!
-                java.util.Arrays.sort(categoryPaths);
-                // Build the metadata string piece now
-                packList(sb,categoryPaths,'+');
-              }
-              else
-              {
-                categoryPaths = specifiedMetadataAttributes;
-                sb.append(metadataString);
-              }
-              
-              String[] actualAcls;
-              String[] denyAcls;
-              
-              String denyAcl;
-              if (acls != null && acls.length == 0)
-              {
-                // No forced acls.  Read the actual acls from livelink, as a set of rights.
-                // We need also to add in support for the special rights objects.  These are:
-                // -1: RIGHT_WORLD
-                // -2: RIGHT_SYSTEM
-                // -3: RIGHT_OWNER
-                // -4: RIGHT_GROUP
-                //
-                // RIGHT_WORLD means guest access.
-                // RIGHT_SYSTEM is "Public Access".
-                // RIGHT_OWNER is access by the owner of the object.
-                // RIGHT_GROUP is access by a member of the base group containing the owner
-                //
-                // These objects are returned by the GetObjectRights() call made above, and NOT
-                // returned by LLUser.ListObjects().  We have to figure out how to map these to
-                // things that are
-                // the equivalent of acls.
-
-                actualAcls = lookupTokens(rights, value);
-                java.util.Arrays.sort(actualAcls);
-                // If security is on, no deny acl is needed for the local authority, since the repository does not support "deny".  But this was added
-                // to be really really really sure.
-                denyAcl = defaultAuthorityDenyToken;
+              int subtype = childrenDocs.toInteger(j, "SubType");
+              boolean childIsFolder = (subtype == LAPI_DOCUMENTS.FOLDERSUBTYPE || subtype == LAPI_DOCUMENTS.PROJECTSUBTYPE ||
+                subtype == LAPI_DOCUMENTS.COMPOUNDDOCUMENTSUBTYPE);
 
-              }
-              else if (acls != null && acls.length > 0)
+              // If it's a folder, we just let it through for now
+              if (!childIsFolder && checkInclude(childrenDocs.toString(j,"Name") + "." + childrenDocs.toString(j,"FileType"), spec) == false)
               {
-                // Forced acls
-                actualAcls = acls;
-                denyAcl = defaultAuthorityDenyToken;
-              }
-              else
-              {
-                // Security is OFF
-                actualAcls = acls;
-                denyAcl = null;
+                if (Logging.connectors.isDebugEnabled())
+                  Logging.connectors.debug("Livelink: Child identifier "+Integer.toString(childID)+" was excluded by inclusion criteria");
+                continue;
               }
 
-              // Now encode the acls.  If null, we write a special value.
-              if (actualAcls == null)
+              if (childIsFolder)
               {
-                sb.append('-');
-                denyAcls = null;
-              }
-              else
-              {
-                sb.append('+');
-                packList(sb,actualAcls,'+');
-                // This was added on 4/21/2008 to support forced acls working with the global default authority.
-                pack(sb,denyAcl,'+');
-                denyAcls = new String[]{denyAcl};
-              }
-
-              // The date does not need to be parseable
-              sb.append(dt.toString());
-
-              // PathNameAttributeVersion comes completely from the spec, so we don't
-              // have to worry about it changing.  No need, therefore, to parse it during
-              // processDocuments.
-              sb.append("=").append(pathNameAttributeVersion);
-
-              // Tack on ingestCgiPath, to insulate us against changes to the repository connection setup.  Added 9/7/07.
-              sb.append("_").append(viewBasePath);
-
-              String versionString = sb.toString();
-              if (Logging.connectors.isDebugEnabled())
-                Logging.connectors.debug("Livelink: Successfully calculated version string for document "+Integer.toString(vol)+":"+Integer.toString(objID)+" : '"+versionString+"'");
-              
-              if (activities.checkDocumentNeedsReindexing(documentIdentifier,versionString))
-              {
-                // Index the document
                 if (Logging.connectors.isDebugEnabled())
-                  Logging.connectors.debug("Livelink: Processing document "+Integer.toString(vol)+":"+Integer.toString(objID));
-                if (checkIngest(llc,objID,spec))
+                  Logging.connectors.debug("Livelink: Child identifier "+Integer.toString(childID)+" is a folder, project, or compound document; adding a reference");
+                if (subtype == LAPI_DOCUMENTS.PROJECTSUBTYPE)
                 {
-                  if (Logging.connectors.isDebugEnabled())
-                    Logging.connectors.debug("Livelink: Decided to ingest document "+Integer.toString(vol)+":"+Integer.toString(objID));
-
-                  // Grab the access tokens for this file from the version string, inside ingest method.
-                  ingestFromLiveLink(llc,documentIdentifier,versionString,actualAcls,denyAcls,categoryPaths,activities,desc,sDesc);
+                  // If we pick up a project object, we need to describe the volume object (which
+                  // will be the root of all documents beneath)
+                  activities.addDocumentReference("F"+new Integer(childID).toString()+":"+new Integer(-childID).toString());
                 }
                 else
-                {
-                  if (Logging.connectors.isDebugEnabled())
-                    Logging.connectors.debug("Livelink: Decided not to ingest document "+Integer.toString(vol)+":"+Integer.toString(objID)+" - Did not match ingestion criteria");
-                  activities.noDocument(documentIdentifier,versionString);
-                }
+                  activities.addDocumentReference("F"+new Integer(vol).toString()+":"+new Integer(childID).toString());
+              }
+              else
+              {
                 if (Logging.connectors.isDebugEnabled())
-                  Logging.connectors.debug("Livelink: Done processing document "+Integer.toString(vol)+":"+Integer.toString(objID));
+                  Logging.connectors.debug("Livelink: Child identifier "+Integer.toString(childID)+" is a simple document; adding a reference");
+
+                activities.addDocumentReference("D"+new Integer(vol).toString()+":"+new Integer(childID).toString());
               }
+
             }
+            break;
           }
-          else
+          catch (InterruptedException e)
           {
-            if (Logging.connectors.isDebugEnabled())
-              Logging.connectors.debug("Livelink: Could not get rights for object "+Integer.toString(vol)+":"+Integer.toString(objID)+" - deleting");
-            activities.deleteDocument(documentIdentifier);
+            t.interrupt();
+            throw new ManifoldCFException("Interrupted: "+e.getMessage(),e,ManifoldCFException.INTERRUPTED);
+          }
+          catch (RuntimeException e)
+          {
+            sanityRetryCount = handleLivelinkRuntimeException(e,sanityRetryCount,true);
             continue;
           }
         }
+        if (Logging.connectors.isDebugEnabled())
+          Logging.connectors.debug("Livelink: Done processing folder "+Integer.toString(vol)+":"+Integer.toString(objID));
+      }
+      else
+      {
+        // === Livelink document ===
+              
+        // The version string includes the following:
+        // 1) The modify date for the document
+        // 2) The rights for the document, ordered (which can change without changing the ModifyDate field)
+        // 3) The requested metadata fields (category and attribute, ordered) for the document
+        //
+        // The document identifiers are object id's.
+  
+        StringBuilder sb = new StringBuilder();
+
+        String[] categoryPaths;
+        if (sDesc.includeAllMetadata())
+        {
+          // Find all the metadata associated with this object, and then
+          // find the set of category pathnames that correspond to it.
+          int[] catIDs = getObjectCategoryIDs(vol,objID);
+          categoryPaths = catAccum.getCategoryPathsAttributeNames(catIDs);
+          // Sort!
+          java.util.Arrays.sort(categoryPaths);
+          // Build the metadata string piece now
+          packList(sb,categoryPaths,'+');
+        }
         else
         {
+          categoryPaths = specifiedMetadataAttributes;
+          sb.append(metadataString);
+        }
+              
+        String[] actualAcls;
+        String[] denyAcls;
+              
+        String denyAcl;
+        if (acls != null && acls.length == 0)
+        {
+          // No forced acls.  Read the actual acls from livelink, as a set of rights.
+          // We need also to add in support for the special rights objects.  These are:
+          // -1: RIGHT_WORLD
+          // -2: RIGHT_SYSTEM
+          // -3: RIGHT_OWNER
+          // -4: RIGHT_GROUP
+          //
+          // RIGHT_WORLD means guest access.
+          // RIGHT_SYSTEM is "Public Access".
+          // RIGHT_OWNER is access by the owner of the object.
+          // RIGHT_GROUP is access by a member of the base group containing the owner
+          //
+          // These objects are returned by the GetObjectRights() call made above, and NOT
+          // returned by LLUser.ListObjects().  We have to figure out how to map these to
+          // things that are
+          // the equivalent of acls.
+
+          actualAcls = lookupTokens(rights, value);
+          java.util.Arrays.sort(actualAcls);
+          // If security is on, no deny acl is needed for the local authority, since the repository does not support "deny".  But this was added
+          // to be really really really sure.
+          denyAcl = defaultAuthorityDenyToken;
+              
+        }
+        else if (acls != null && acls.length > 0)
+        {
+          // Forced acls
+          actualAcls = acls;
+          denyAcl = defaultAuthorityDenyToken;
+        }
+        else
+        {
+          // Security is OFF
+          actualAcls = acls;
+          denyAcl = null;
+        }
+
+        // Now encode the acls.  If null, we write a special value.
+        if (actualAcls == null)
+        {
+          sb.append('-');
+          denyAcls = null;
+        }
+        else
+        {
+          sb.append('+');
+          packList(sb,actualAcls,'+');
+          // This was added on 4/21/2008 to support forced acls working with the global default authority.
+          pack(sb,denyAcl,'+');
+          denyAcls = new String[]{denyAcl};
+        }
+
+        // The date does not need to be parseable
+        sb.append(new Long(dt.getTime()).toString());
+
+        // PathNameAttributeVersion comes completely from the spec, so we don't
+        // have to worry about it changing.  No need, therefore, to parse it during
+        // processDocuments.
+        sb.append("=").append(pathNameAttributeVersion);
+          
+        // Tack on ingestCgiPath, to insulate us against changes to the repository connection setup.  Added 9/7/07.
+        sb.append("_").append(viewBasePath);
+
+        String versionString = sb.toString();
+        if (Logging.connectors.isDebugEnabled())
+          Logging.connectors.debug("Livelink: Successfully calculated version string for document "+Integer.toString(vol)+":"+Integer.toString(objID)+" : '"+versionString+"'");
+              
+        if (!activities.checkDocumentNeedsReindexing(documentIdentifier,versionString))
+          continue;
+        
+        // Index the document
+        if (Logging.connectors.isDebugEnabled())
+          Logging.connectors.debug("Livelink: Processing document "+Integer.toString(vol)+":"+Integer.toString(objID));
+        if (!checkIngest(llc,objID,spec))
+        {
           if (Logging.connectors.isDebugEnabled())
-            Logging.connectors.debug("Livelink: Crawl user cannot see contents of object "+Integer.toString(vol)+":"+Integer.toString(objID)+" - deleting");
-          activities.deleteDocument(documentIdentifier);
+            Logging.connectors.debug("Livelink: Decided not to ingest document "+Integer.toString(vol)+":"+Integer.toString(objID)+" - Did not match ingestion criteria");
+          activities.noDocument(documentIdentifier,versionString);
           continue;
         }
-      }
-      else
-      {
+
         if (Logging.connectors.isDebugEnabled())
-          Logging.connectors.debug("Livelink: Object "+Integer.toString(vol)+":"+Integer.toString(objID)+" has no information - deleting");
-        activities.deleteDocument(documentIdentifier);
-        continue;
-      }
+          Logging.connectors.debug("Livelink: Decided to ingest document "+Integer.toString(vol)+":"+Integer.toString(objID));
 
+        // Grab the access tokens for this file from the version string, inside ingest method.
+        ingestFromLiveLink(llc,documentIdentifier,versionString,actualAcls,denyAcls,categoryPaths,activities,desc,sDesc);
+          
+        if (Logging.connectors.isDebugEnabled())
+          Logging.connectors.debug("Livelink: Done processing document "+Integer.toString(vol)+":"+Integer.toString(objID));
+      }
     }
   }
 
@@ -4171,20 +4165,12 @@ public class LivelinkConnector extends o
 
     String contextMsg = "for '"+documentIdentifier+"'";
 
-    String viewHttpAddress = convertToViewURI(documentIdentifier);
-    if (viewHttpAddress == null)
-    {
-      if (Logging.connectors.isDebugEnabled())
-        Logging.connectors.debug("Livelink: No view URI "+contextMsg+" - not ingesting");
-      return;
-    }
 
     // Fetch logging
     long startTime = System.currentTimeMillis();
-    String resultCode = "FAILED";
+    String resultCode = null;
     String resultDescription = null;
     Long readSize = null;
-    boolean wasInterrupted = false;
     int objID;
     int vol;
 
@@ -4204,14 +4190,25 @@ public class LivelinkConnector extends o
     // Try/finally for fetch logging
     try
     {
+      String viewHttpAddress = convertToViewURI(documentIdentifier);
+      if (viewHttpAddress == null)
+      {
+        if (Logging.connectors.isDebugEnabled())
+          Logging.connectors.debug("Livelink: No view URI "+contextMsg+" - not ingesting");
+        resultCode = "NOVIEWURI";
+        resultDescription = "Document had no view URI";
+        activities.noDocument(documentIdentifier,version);
+        return;
+      }
+      
       // Check URL first
       if (!activities.checkURLIndexable(viewHttpAddress))
       {
         // Document not ingestable due to URL
+        resultCode = activities.EXCLUDED_URL;
         resultDescription = "URL ("+viewHttpAddress+") was rejected by output connector";
         if (Logging.connectors.isDebugEnabled())
           Logging.connectors.debug("Livelink: Excluding document "+documentIdentifier+" because its URL ("+viewHttpAddress+") was rejected by output connector");
-        resultCode = "URLEXCLUSION";
         activities.noDocument(documentIdentifier,version);
         return;
       }
@@ -4222,13 +4219,17 @@ public class LivelinkConnector extends o
       if (!objInfo.exists())
       {
         resultCode = "OBJECTNOTFOUND";
+        resultDescription = "Object was not found in Livelink";
         Logging.connectors.debug("Livelink: No object "+contextMsg+": not ingesting");
+        activities.noDocument(documentIdentifier,version);
         return;
       }
       if (!versInfo.exists())
       {
         resultCode = "VERSIONNOTFOUND";
+        resultDescription = "Version was not found in Livelink";
         Logging.connectors.debug("Livelink: No version data "+contextMsg+": not ingesting");
+        activities.noDocument(documentIdentifier,version);
         return;
       }
 
@@ -4236,10 +4237,10 @@ public class LivelinkConnector extends o
       if (!activities.checkMimeTypeIndexable(mimeType))
       {
         // Document not indexable because of its mime type
+        resultCode = activities.EXCLUDED_MIMETYPE;
         resultDescription = "Mime type ("+mimeType+") was rejected by output connector";
         if (Logging.connectors.isDebugEnabled())
           Logging.connectors.debug("Livelink: Excluding document "+documentIdentifier+" because its mime type ("+mimeType+") was rejected by output connector");
-        resultCode = "MIMETYPEEXCLUSION";
         activities.noDocument(documentIdentifier,version);
         return;
       }
@@ -4248,10 +4249,10 @@ public class LivelinkConnector extends o
       if (dataSize == null)
       {
         // Document had no length
-        resultDescription = "Document had no length";
+        resultCode = "DOCUMENTNOLENGTH";
+        resultDescription = "Document had no length in Livelink";
         if (Logging.connectors.isDebugEnabled())
           Logging.connectors.debug("Livelink: Excluding document "+documentIdentifier+" because it had no length");
-        resultCode = "DOCUMENTNOLENGTH";
         activities.noDocument(documentIdentifier,version);
         return;
       }
@@ -4259,10 +4260,10 @@ public class LivelinkConnector extends o
       if (!activities.checkLengthIndexable(dataSize.longValue()))
       {
         // Document not indexable because of its length
+        resultCode = activities.EXCLUDED_LENGTH;
         resultDescription = "Document length ("+dataSize+") was rejected by output connector";
         if (Logging.connectors.isDebugEnabled())
           Logging.connectors.debug("Livelink: Excluding document "+documentIdentifier+" because its length ("+dataSize+") was rejected by output connector");
-        resultCode = "DOCUMENTTOOLONG";
         activities.noDocument(documentIdentifier,version);
         return;
       }
@@ -4271,10 +4272,10 @@ public class LivelinkConnector extends o
       if (!activities.checkDateIndexable(modifyDate))
       {
         // Document not indexable because of its date
+        resultCode = activities.EXCLUDED_DATE;
         resultDescription = "Document date ("+modifyDate+") was rejected by output connector";
         if (Logging.connectors.isDebugEnabled())
           Logging.connectors.debug("Livelink: Excluding document "+documentIdentifier+" because its date ("+modifyDate+") was rejected by output connector");
-        resultCode = "DOCUMENTBADDATE";
         activities.noDocument(documentIdentifier,version);
         return;
       }
@@ -4368,232 +4369,154 @@ public class LivelinkConnector extends o
       {
         // Use HTTP to fetch document!
         String ingestHttpAddress = convertToIngestURI(documentIdentifier);
-        if (ingestHttpAddress != null)
+        if (ingestHttpAddress == null)
         {
+          if (Logging.connectors.isDebugEnabled())
+            Logging.connectors.debug("Livelink: No fetch URI "+contextMsg+" - not ingesting");
+          resultCode = "NOURI";
+          resultDescription = "Document had no fetch URI";
+          activities.noDocument(documentIdentifier,version);
+          return;
+        }
 
-          // Set up connection
-          HttpClient client = getInitializedClient(contextMsg);
+        // Set up connection
+        HttpClient client = getInitializedClient(contextMsg);
 
-          long currentTime;
+        long currentTime;
 
-          if (Logging.connectors.isInfoEnabled())
-            Logging.connectors.info("Livelink: " + ingestHttpAddress);
+        if (Logging.connectors.isInfoEnabled())
+          Logging.connectors.info("Livelink: " + ingestHttpAddress);
 
 
-          HttpGet method = new HttpGet(getHost().toURI() + ingestHttpAddress);
-          method.setHeader(new BasicHeader("Accept","*/*"));
+        HttpGet method = new HttpGet(getHost().toURI() + ingestHttpAddress);
+        method.setHeader(new BasicHeader("Accept","*/*"));
 
-          ExecuteMethodThread methodThread = new ExecuteMethodThread(client,method);
-          methodThread.start();
-          try
+        boolean wasInterrupted = false;
+        ExecuteMethodThread methodThread = new ExecuteMethodThread(client,method);
+        methodThread.start();
+        try
+        {
+          int statusCode = methodThread.getResponseCode();
+          switch (statusCode)
           {
+          case 500:
+          case 502:
+            Logging.connectors.warn("Livelink: Service interruption during fetch "+contextMsg+" with Livelink HTTP Server, retrying...");
+            resultCode = "FETCHFAILED";
+            resultDescription = "HTTP error code "+statusCode+" fetching document";
+            throw new ServiceInterruption("Service interruption during fetch",new ManifoldCFException(Integer.toString(statusCode)+" error while fetching"),System.currentTimeMillis()+60000L,
+              System.currentTimeMillis()+600000L,-1,true);
+
+          case HttpStatus.SC_UNAUTHORIZED:
+            Logging.connectors.warn("Livelink: Document fetch unauthorized for "+ingestHttpAddress+" ("+contextMsg+")");
+            // Since we logged in, we should fail here if the ingestion user doesn't have access to the
+            // the document, but if we do, don't fail hard.
+            resultCode = "UNAUTHORIZED";
+            resultDescription = "Document fetch was unauthorized by IIS";
+            activities.noDocument(documentIdentifier,version);
+            return;
 
-            int statusCode = methodThread.getResponseCode();
-            switch (statusCode)
+          case HttpStatus.SC_OK:
+            if (Logging.connectors.isDebugEnabled())
+              Logging.connectors.debug("Livelink: Created http document connection to Livelink "+contextMsg);
+            // A non-existent content length will cause a value of -1 to be returned.  This seems to indicate that the session login did not work right.
+            if (methodThread.getResponseContentLength() < 0)
             {
-            case 500:
-            case 502:
-              Logging.connectors.warn("Livelink: Service interruption during fetch "+contextMsg+" with Livelink HTTP Server, retrying...");
-              throw new ServiceInterruption("Service interruption during fetch",new ManifoldCFException(Integer.toString(statusCode)+" error while fetching"),System.currentTimeMillis()+60000L,
-                System.currentTimeMillis()+600000L,-1,true);
-
-            case HttpStatus.SC_UNAUTHORIZED:
-              Logging.connectors.warn("Livelink: Document fetch unauthorized for "+ingestHttpAddress+" ("+contextMsg+")");
-              // Since we logged in, we should fail here if the ingestion user doesn't have access to the
-              // the document, but if we do, don't fail hard.
-              resultCode = "UNAUTHORIZED";
+              resultCode = "SESSIONLOGINFAILED";
+              resultDescription = "Response content length was -1, which usually means session login did not succeed";
               activities.noDocument(documentIdentifier,version);
               return;
-
-            case HttpStatus.SC_OK:
-              if (Logging.connectors.isDebugEnabled())
-                Logging.connectors.debug("Livelink: Created http document connection to Livelink "+contextMsg);
-              // A non-existent content length will cause a value of -1 to be returned.  This seems to indicate that the session login did not work right.
-              if (methodThread.getResponseContentLength() >= 0)
+            }
+              
+            try
+            {
+              InputStream is = methodThread.getSafeInputStream();
+              try
               {
-                try
-                {
-                  InputStream is = methodThread.getSafeInputStream();
-                  try
-                  {
-                    rd.setBinary(is,dataSize);
+                rd.setBinary(is,dataSize);
                             
-                    activities.ingestDocumentWithException(documentIdentifier,version,viewHttpAddress,rd);
+                activities.ingestDocumentWithException(documentIdentifier,version,viewHttpAddress,rd);
+                resultCode = "OK";
+                readSize = dataSize;
                     
-                    if (Logging.connectors.isDebugEnabled())
-                      Logging.connectors.debug("Livelink: Ingesting done "+contextMsg);
+                if (Logging.connectors.isDebugEnabled())
+                  Logging.connectors.debug("Livelink: Ingesting done "+contextMsg);
 
-                  }
-                  finally
-                  {
-                    // Close stream via thread, since otherwise this can hang
-                    is.close();
-                  }
-                }
-                catch (java.net.SocketTimeoutException e)
-                {
-                  resultCode = "DATATIMEOUT";
-                  resultDescription = e.getMessage();
-                  currentTime = System.currentTimeMillis();
-                  Logging.connectors.warn("Livelink: Livelink socket timed out ingesting from the Livelink HTTP Server "+contextMsg+": "+e.getMessage(), e);
-                  throw new ServiceInterruption("Socket timed out: "+e.getMessage(),e,currentTime+300000L,currentTime+6*3600000L,-1,false);
-                }
-                catch (java.net.SocketException e)
-                {
-                  resultCode = "DATASOCKETERROR";
-                  resultDescription = e.getMessage();
-                  currentTime = System.currentTimeMillis();
-                  Logging.connectors.warn("Livelink: Livelink socket error ingesting from the Livelink HTTP Server "+contextMsg+": "+e.getMessage(), e);
-                  throw new ServiceInterruption("Socket error: "+e.getMessage(),e,currentTime+300000L,currentTime+6*3600000L,-1,false);
-                }
-                catch (javax.net.ssl.SSLHandshakeException e)
-                {
-                  resultCode = "DATASSLHANDSHAKEERROR";
-                  resultDescription = e.getMessage();
-                  currentTime = System.currentTimeMillis();
-                  Logging.connectors.warn("Livelink: SSL handshake failed authenticating "+contextMsg+": "+e.getMessage(),e);
-                  throw new ServiceInterruption("SSL handshake error: "+e.getMessage(),e,currentTime+60000L,currentTime+300000L,-1,true);
-                }
-                catch (ConnectTimeoutException e)
-                {
-                  resultCode = "CONNECTTIMEOUT";
-                  resultDescription = e.getMessage();
-                  currentTime = System.currentTimeMillis();
-                  Logging.connectors.warn("Livelink: Livelink socket timed out connecting to the Livelink HTTP Server "+contextMsg+": "+e.getMessage(), e);
-                  throw new ServiceInterruption("Connect timed out: "+e.getMessage(),e,currentTime+300000L,currentTime+6*3600000L,-1,false);
-                }
-                catch (InterruptedException e)
-                {
-                  wasInterrupted = true;
-                  throw new ManifoldCFException("Interrupted: "+e.getMessage(),e,ManifoldCFException.INTERRUPTED);
-                }
-                catch (InterruptedIOException e)
-                {
-                  wasInterrupted = true;
-                  throw new ManifoldCFException("Interrupted: "+e.getMessage(),e,ManifoldCFException.INTERRUPTED);
-                }
-                catch (HttpException e)
-                {
-                  resultCode = "HTTPEXCEPTION";
-                  resultDescription = e.getMessage();
-                  // Treat unknown error ingesting data as a transient condition
-                  currentTime = System.currentTimeMillis();
-                  Logging.connectors.warn("Livelink: HTTP exception ingesting "+contextMsg+": "+e.getMessage(),e);
-                  throw new ServiceInterruption("HTTP exception ingesting "+contextMsg+": "+e.getMessage(),e,currentTime+300000L,currentTime+6*3600000L,-1,false);
-                }
-                catch (IOException e)
-                {
-                  resultCode = "DATAEXCEPTION";
-                  resultDescription = e.getMessage();
-                  // Treat unknown error ingesting data as a transient condition
-                  currentTime = System.currentTimeMillis();
-                  Logging.connectors.warn("Livelink: IO exception ingesting "+contextMsg+": "+e.getMessage(),e);
-                  throw new ServiceInterruption("IO exception ingesting "+contextMsg+": "+e.getMessage(),e,currentTime+300000L,currentTime+6*3600000L,-1,false);
-                }
-                readSize = dataSize;
               }
-              else
+              finally
               {
-                resultCode = "SESSIONLOGINFAILED";
-                activities.noDocument(documentIdentifier,version);
+                // Close stream via thread, since otherwise this can hang
+                is.close();
               }
-              break;
-            case HttpStatus.SC_BAD_REQUEST:
-            case HttpStatus.SC_USE_PROXY:
-            case HttpStatus.SC_GONE:
-              resultCode = "ERROR "+Integer.toString(statusCode);
-              throw new ManifoldCFException("Unrecoverable request failure; error = "+Integer.toString(statusCode));
-            default:
-              resultCode = "UNKNOWN";
-              Logging.connectors.warn("Livelink: Attempt to retrieve document from '"+ingestHttpAddress+"' received a response of "+Integer.toString(statusCode)+"; retrying in one minute");
-              currentTime = System.currentTimeMillis();
-              throw new ServiceInterruption("Fetch failed; retrying in 1 minute",new ManifoldCFException("Fetch failed with unknown code "+Integer.toString(statusCode)),
-                currentTime+60000L,currentTime+600000L,-1,true);
             }
-          }
-          catch (InterruptedException e)
-          {
-            // Drop the connection on the floor
-            methodThread.interrupt();
-            methodThread = null;
-            throw new ManifoldCFException("Interrupted: "+e.getMessage(),e,ManifoldCFException.INTERRUPTED);
-          }
-          catch (java.net.SocketTimeoutException e)
-          {
-            Logging.connectors.warn("Livelink: Socket timed out reading from the Livelink HTTP Server "+contextMsg+": "+e.getMessage(), e);
-            resultCode = "TIMEOUT";
-            resultDescription = e.getMessage();
-            currentTime = System.currentTimeMillis();
-            throw new ServiceInterruption("Socket timed out: "+e.getMessage(),e,currentTime+300000L,currentTime+6*3600000L,-1,true);
-          }
-          catch (java.net.SocketException e)
-          {
-            Logging.connectors.warn("Livelink: Socket error reading from Livelink HTTP Server "+contextMsg+": "+e.getMessage(), e);
-            resultCode = "SOCKETERROR";
-            resultDescription = e.getMessage();
-            currentTime = System.currentTimeMillis();
-            throw new ServiceInterruption("Socket error: "+e.getMessage(),e,currentTime+300000L,currentTime+6*3600000L,-1,true);
-          }
-          catch (javax.net.ssl.SSLHandshakeException e)
-          {
-            currentTime = System.currentTimeMillis();
-            Logging.connectors.warn("Livelink: SSL handshake failed "+contextMsg+": "+e.getMessage(),e);
-            resultCode = "SSLHANDSHAKEERROR";
-            resultDescription = e.getMessage();
-            throw new ServiceInterruption("SSL handshake error: "+e.getMessage(),e,currentTime+60000L,currentTime+300000L,-1,true);
-          }
-          catch (ConnectTimeoutException e)
-          {
-            Logging.connectors.warn("Livelink: Connect timed out reading from the Livelink HTTP Server "+contextMsg+": "+e.getMessage(), e);
-            resultCode = "CONNECTTIMEOUT";
-            resultDescription = e.getMessage();
+            catch (InterruptedException e)
+            {
+              wasInterrupted = true;
+              throw new ManifoldCFException("Interrupted: "+e.getMessage(),e,ManifoldCFException.INTERRUPTED);
+            }
+            catch (HttpException e)
+            {
+              resultCode = e.getClass().getSimpleName().toUpperCase(Locale.ROOT);
+              resultDescription = e.getMessage();
+              handleHttpException(contextMsg,e);
+            }
+            catch (IOException e)
+            {
+              resultCode = e.getClass().getSimpleName().toUpperCase(Locale.ROOT);
+              resultDescription = e.getMessage();
+              handleIOException(contextMsg,e);
+            }
+            break;
+          case HttpStatus.SC_BAD_REQUEST:
+          case HttpStatus.SC_USE_PROXY:
+          case HttpStatus.SC_GONE:
+            resultCode = "HTTPERROR";
+            resultDescription = "Http request returned status "+Integer.toString(statusCode);
+            throw new ManifoldCFException("Unrecoverable request failure; error = "+Integer.toString(statusCode));
+          default:
+            resultCode = "UNKNOWNHTTPCODE";
+            resultDescription = "Http request returned status "+Integer.toString(statusCode);
+            Logging.connectors.warn("Livelink: Attempt to retrieve document from '"+ingestHttpAddress+"' received a response of "+Integer.toString(statusCode)+"; retrying in one minute");
             currentTime = System.currentTimeMillis();
-            throw new ServiceInterruption("Connect timed out: "+e.getMessage(),e,currentTime+300000L,currentTime+6*3600000L,-1,true);
-          }
-          catch (InterruptedIOException e)
-          {
-            methodThread.interrupt();
-            throw new ManifoldCFException("Interrupted: "+e.getMessage(),e,ManifoldCFException.INTERRUPTED);
-          }
-          catch (HttpException e)
-          {
-            resultCode = "EXCEPTION";
-            resultDescription = e.getMessage();
-            throw new ManifoldCFException("Exception getting response "+contextMsg+": "+e.getMessage(), e);
+            throw new ServiceInterruption("Fetch failed; retrying in 1 minute",new ManifoldCFException("Fetch failed with unknown code "+Integer.toString(statusCode)),
+              currentTime+60000L,currentTime+600000L,-1,true);
           }
-          catch (IOException e)
-          {
-            resultCode = "EXCEPTION";
-            resultDescription = e.getMessage();
-            throw new ManifoldCFException("Exception getting response "+contextMsg+": "+e.getMessage(), e);
-          }
-          finally
+        }
+        catch (InterruptedException e)
+        {
+          // Drop the connection on the floor
+          methodThread.interrupt();
+          methodThread = null;
+          throw new ManifoldCFException("Interrupted: "+e.getMessage(),e,ManifoldCFException.INTERRUPTED);
+        }
+        catch (HttpException e)
+        {
+          resultCode = e.getClass().getSimpleName().toUpperCase(Locale.ROOT);
+          resultDescription = e.getMessage();
+          handleHttpException(contextMsg,e);
+        }
+        catch (IOException e)
+        {
+          resultCode = e.getClass().getSimpleName().toUpperCase(Locale.ROOT);
+          resultDescription = e.getMessage();
+          handleIOException(contextMsg,e);
+        }
+        finally
+        {
+          if (methodThread != null)
           {
-            if (methodThread != null)
+            methodThread.abort();
+            try
             {
-              methodThread.abort();
               if (!wasInterrupted)
-              {
-                try
-                {
-                  methodThread.finishUp();
-                }
-                catch (InterruptedException e)
-                {
-                  wasInterrupted = true;
-                  throw new ManifoldCFException(e.getMessage(),e,ManifoldCFException.INTERRUPTED);
-                }
-              }
+                methodThread.finishUp();
+            }
+            catch (InterruptedException e)
+            {
+              throw new ManifoldCFException(e.getMessage(),e,ManifoldCFException.INTERRUPTED);
             }
           }
         }
-        else
-        {
-          if (Logging.connectors.isDebugEnabled())
-            Logging.connectors.debug("Livelink: No fetch URI "+contextMsg+" - not ingesting");
-          resultCode = "NOURI";
-          return;
-        }
       }
       else
       {
@@ -4602,9 +4525,10 @@ public class LivelinkConnector extends o
               
         // Fire up the document reading thread
         DocumentReadingThread t = new DocumentReadingThread(vol,objID,0);
+        boolean wasInterrupted = false;
+        t.start();
         try 
         {
-          t.start();
           try
           {
             InputStream is = t.getSafeInputStream();
@@ -4613,18 +4537,14 @@ public class LivelinkConnector extends o
               // Can only index while background thread is running!
               rd.setBinary(is, dataSize);
               activities.ingestDocumentWithException(documentIdentifier, version, viewHttpAddress, rd);
+              resultCode = "OK";
+              readSize = dataSize;
             }
             finally
             {
               is.close();
             }
           }
-          catch (ManifoldCFException e)
-          {
-            if (e.getErrorCode() == ManifoldCFException.INTERRUPTED)
-              wasInterrupted = true;
-            throw e;
-          }
           catch (java.net.SocketTimeoutException e)
           {
             throw e;
@@ -4641,8 +4561,6 @@ public class LivelinkConnector extends o
           }
 
           // No errors.  Record the fact that we made it.
-          resultCode = "OK";
-          readSize = dataSize;
         }
         catch (InterruptedException e) 
         {
@@ -4650,49 +4568,73 @@ public class LivelinkConnector extends o
           throw new ManifoldCFException("Interrupted: " + e.getMessage(), e,
             ManifoldCFException.INTERRUPTED);
         }
-        catch (ConnectTimeoutException e)
-        {
-          Logging.connectors.warn("Livelink: Connect timed out "+contextMsg+": "+e.getMessage(), e);
-          resultCode = "CONNECTTIMEOUT";
-          resultDescription = e.getMessage();
-          currentTime = System.currentTimeMillis();
-          throw new ServiceInterruption("Connect timed out: "+e.getMessage(),e,currentTime+300000L,currentTime+6*3600000L,-1,true);
-        }
-        catch (InterruptedIOException e)
-        {
-          t.interrupt();
-          throw new ManifoldCFException("Interrupted: "+e.getMessage(),e,ManifoldCFException.INTERRUPTED);
-        }
         catch (IOException e)
         {
-          resultCode = "EXCEPTION";
+          resultCode = e.getClass().getSimpleName().toUpperCase(Locale.ROOT);
           resultDescription = e.getMessage();
-          throw new ManifoldCFException("Exception getting response "+contextMsg+": "+e.getMessage(), e);
-        }
-        catch (ManifoldCFException e)
-        {
-          if (e.getErrorCode() != ManifoldCFException.INTERRUPTED)
-          {
-            resultCode = "EXCEPTION";
-            resultDescription = e.getMessage();
-          }
-          throw e;
+          handleIOException(contextMsg,e);
         }
         catch (RuntimeException e)
         {
-          resultCode = "EXCEPTION";
+          resultCode = e.getClass().getSimpleName().toUpperCase(Locale.ROOT);
           resultDescription = e.getMessage();
           handleLivelinkRuntimeException(e,0,true);
         }
       }
     }
+    catch (ManifoldCFException e)
+    {
+      if (e.getErrorCode() == ManifoldCFException.INTERRUPTED)
+        resultCode = null;
+      throw e;
+    }
     finally
     {
-      if (!wasInterrupted)
+      if (resultCode != null)
         activities.recordActivity(new Long(startTime),ACTIVITY_FETCH,readSize,vol+":"+objID,resultCode,resultDescription,null);
     }
   }
 
+  protected static void handleHttpException(String contextMsg, HttpException e)
+    throws ManifoldCFException, ServiceInterruption
+  {
+    long currentTime = System.currentTimeMillis();
+    // Treat unknown error ingesting data as a transient condition
+    Logging.connectors.warn("Livelink: HTTP exception ingesting "+contextMsg+": "+e.getMessage(),e);
+    throw new ServiceInterruption("HTTP exception ingesting "+contextMsg+": "+e.getMessage(),e,currentTime+300000L,currentTime+6*3600000L,-1,false);
+  }
+  
+  protected static void handleIOException(String contextMsg, IOException e)
+    throws ManifoldCFException, ServiceInterruption
+  {
+    long currentTime = System.currentTimeMillis();
+    if (e instanceof java.net.SocketTimeoutException)
+    {
+      Logging.connectors.warn("Livelink: Livelink socket timed out ingesting from the Livelink HTTP Server "+contextMsg+": "+e.getMessage(), e);
+      throw new ServiceInterruption("Socket timed out: "+e.getMessage(),e,currentTime+300000L,currentTime+6*3600000L,-1,false);
+    }
+    if (e instanceof java.net.SocketException)
+    {
+      Logging.connectors.warn("Livelink: Livelink socket error ingesting from the Livelink HTTP Server "+contextMsg+": "+e.getMessage(), e);
+      throw new ServiceInterruption("Socket error: "+e.getMessage(),e,currentTime+300000L,currentTime+6*3600000L,-1,false);
+    }
+    if (e instanceof javax.net.ssl.SSLHandshakeException)
+    {
+      Logging.connectors.warn("Livelink: SSL handshake failed authenticating "+contextMsg+": "+e.getMessage(),e);
+      throw new ServiceInterruption("SSL handshake error: "+e.getMessage(),e,currentTime+60000L,currentTime+300000L,-1,true);
+    }
+    if (e instanceof ConnectTimeoutException)
+    {
+      Logging.connectors.warn("Livelink: Livelink socket timed out connecting to the Livelink HTTP Server "+contextMsg+": "+e.getMessage(), e);
+      throw new ServiceInterruption("Connect timed out: "+e.getMessage(),e,currentTime+300000L,currentTime+6*3600000L,-1,false);
+    }
+    if (e instanceof InterruptedIOException)
+      throw new ManifoldCFException("Interrupted: "+e.getMessage(),e,ManifoldCFException.INTERRUPTED);
+    // Treat unknown error ingesting data as a transient condition
+    Logging.connectors.warn("Livelink: IO exception ingesting "+contextMsg+": "+e.getMessage(),e);
+    throw new ServiceInterruption("IO exception ingesting "+contextMsg+": "+e.getMessage(),e,currentTime+300000L,currentTime+6*3600000L,-1,false);
+  }
+  
   /** Initialize a livelink client connection */
   protected HttpClient getInitializedClient(String contextMsg)
     throws ServiceInterruption, ManifoldCFException



Mime
View raw message