manifoldcf-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kwri...@apache.org
Subject svn commit: r1630136 - /manifoldcf/branches/CONNECTORS-1067/connectors/livelink/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/livelink/LivelinkConnector.java
Date Wed, 08 Oct 2014 15:15:50 GMT
Author: kwright
Date: Wed Oct  8 15:15:50 2014
New Revision: 1630136

URL: http://svn.apache.org/r1630136
Log:
Update livelink connector

Modified:
    manifoldcf/branches/CONNECTORS-1067/connectors/livelink/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/livelink/LivelinkConnector.java

Modified: manifoldcf/branches/CONNECTORS-1067/connectors/livelink/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/livelink/LivelinkConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1067/connectors/livelink/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/livelink/LivelinkConnector.java?rev=1630136&r1=1630135&r2=1630136&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-1067/connectors/livelink/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/livelink/LivelinkConnector.java
(original)
+++ manifoldcf/branches/CONNECTORS-1067/connectors/livelink/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/livelink/LivelinkConnector.java
Wed Oct  8 15:15:50 2014
@@ -4185,466 +4185,485 @@ public class LivelinkConnector extends o
     try
     {
       // Check URL first
-      if (activities.checkURLIndexable(viewHttpAddress))
+      if (!activities.checkURLIndexable(viewHttpAddress))
       {
+        // Document not ingestable due to URL
+        resultDescription = "URL ("+viewHttpAddress+") was rejected by output connector";
+        if (Logging.connectors.isDebugEnabled())
+          Logging.connectors.debug("Livelink: Excluding document "+documentIdentifier+" because
its URL ("+viewHttpAddress+") was rejected by output connector");
+        resultCode = "URLEXCLUSION";
+        activities.noDocument(documentIdentifier,version);
+        return;
+      }
+      
+      // Add general metadata
+      ObjectInformation objInfo = llc.getObjectInformation(vol,objID);
+      VersionInformation versInfo = llc.getVersionInformation(vol,objID,0);
+      if (!objInfo.exists())
+      {
+        resultCode = "OBJECTNOTFOUND";
+        Logging.connectors.debug("Livelink: No object "+contextMsg+": not ingesting");
+        return;
+      }
+      if (!versInfo.exists())
+      {
+        resultCode = "VERSIONNOTFOUND";
+        Logging.connectors.debug("Livelink: No version data "+contextMsg+": not ingesting");
+        return;
+      }
 
-        // Add general metadata
-        ObjectInformation objInfo = llc.getObjectInformation(vol,objID);
-        VersionInformation versInfo = llc.getVersionInformation(vol,objID,0);
-        if (!objInfo.exists())
-        {
-          resultCode = "OBJECTNOTFOUND";
-          Logging.connectors.debug("Livelink: No object "+contextMsg+": not ingesting");
-          return;
-        }
-        if (!versInfo.exists())
-        {
-          resultCode = "VERSIONNOTFOUND";
-          Logging.connectors.debug("Livelink: No version data "+contextMsg+": not ingesting");
-          return;
-        }
+      String mimeType = versInfo.getMimeType();
+      if (!activities.checkMimeTypeIndexable(mimeType))
+      {
+        // Document not indexable because of its mime type
+        resultDescription = "Mime type ("+mimeType+") was rejected by output connector";
+        if (Logging.connectors.isDebugEnabled())
+          Logging.connectors.debug("Livelink: Excluding document "+documentIdentifier+" because
its mime type ("+mimeType+") was rejected by output connector");
+        resultCode = "MIMETYPEEXCLUSION";
+        activities.noDocument(documentIdentifier,version);
+        return;
+      }
+        
+      Long dataSize = versInfo.getDataSize();
+      if (dataSize == null)
+      {
+        // Document had no length
+        resultDescription = "Document had no length";
+        if (Logging.connectors.isDebugEnabled())
+          Logging.connectors.debug("Livelink: Excluding document "+documentIdentifier+" because
it had no length");
+        resultCode = "DOCUMENTNOLENGTH";
+        activities.noDocument(documentIdentifier,version);
+        return;
+      }
+      
+      if (!activities.checkLengthIndexable(dataSize.longValue()))
+      {
+        // Document not indexable because of its length
+        resultDescription = "Document length ("+dataSize+") was rejected by output connector";
+        if (Logging.connectors.isDebugEnabled())
+          Logging.connectors.debug("Livelink: Excluding document "+documentIdentifier+" because
its length ("+dataSize+") was rejected by output connector");
+        resultCode = "DOCUMENTTOOLONG";
+        activities.noDocument(documentIdentifier,version);
+        return;
+      }
 
-        String mimeType = versInfo.getMimeType();
-        if (activities.checkMimeTypeIndexable(mimeType))
-        {
-          Long dataSize = versInfo.getDataSize();
-          if (dataSize != null && activities.checkLengthIndexable(dataSize.longValue()))
-          {
-            String fileName = versInfo.getFileName();
-            Date creationDate = objInfo.getCreationDate();
-            Date modifyDate = versInfo.getModifyDate();
-            Integer parentID = objInfo.getParentId();
-            RepositoryDocument rd = new RepositoryDocument();
+      Date modifyDate = versInfo.getModifyDate();
+      if (!activities.checkDateIndexable(modifyDate))
+      {
+        // Document not indexable because of its date
+        resultDescription = "Document date ("+modifyDate+") was rejected by output connector";
+        if (Logging.connectors.isDebugEnabled())
+          Logging.connectors.debug("Livelink: Excluding document "+documentIdentifier+" because
its date ("+modifyDate+") was rejected by output connector");
+        resultCode = "DOCUMENTBADDATE";
+        activities.noDocument(documentIdentifier,version);
+        return;
+      }
+      
+      String fileName = versInfo.getFileName();
+      Date creationDate = objInfo.getCreationDate();
+      Integer parentID = objInfo.getParentId();
+      
+      
+      RepositoryDocument rd = new RepositoryDocument();
 
+      // Add general data we need for the output connector
+      if (mimeType != null)
+        rd.setMimeType(mimeType);
+      if (fileName != null)
+        rd.setFileName(fileName);
+      if (creationDate != null)
+        rd.setCreatedDate(creationDate);
+      if (modifyDate != null)
+        rd.setModifiedDate(modifyDate);
             
-            // Add general data we need for the output connector
-            if (mimeType != null)
-              rd.setMimeType(mimeType);
-            if (fileName != null)
-              rd.setFileName(fileName);
-            if (creationDate != null)
-              rd.setCreatedDate(creationDate);
-            if (modifyDate != null)
-              rd.setModifiedDate(modifyDate);
-            
-            rd.addField(GENERAL_NAME_FIELD,objInfo.getName());
-            rd.addField(GENERAL_DESCRIPTION_FIELD,objInfo.getComments());
-            if (creationDate != null)
-              rd.addField(GENERAL_CREATIONDATE_FIELD,new Long(creationDate.getTime()).toString());
-            if (modifyDate != null)
-              rd.addField(GENERAL_MODIFYDATE_FIELD,new Long(modifyDate.getTime()).toString());
-            if (parentID != null)
-              rd.addField(GENERAL_PARENTID,parentID.toString());
-            UserInformation owner = llc.getUserInformation(objInfo.getOwnerId().intValue());
-            UserInformation creator = llc.getUserInformation(objInfo.getCreatorId().intValue());
-            UserInformation modifier = llc.getUserInformation(versInfo.getOwnerId().intValue());
-            if (owner != null)
-              rd.addField(GENERAL_OWNER,owner.getName());
-            if (creator != null)
-              rd.addField(GENERAL_CREATOR,creator.getName());
-            if (modifier != null)
-              rd.addField(GENERAL_MODIFIER,modifier.getName());
-
-            // Iterate over the metadata items.  These are organized by category
-            // for speed of lookup.
-
-            Iterator<MetadataItem> catIter = desc.getItems(categoryPaths);
-            while (catIter.hasNext())
+      rd.addField(GENERAL_NAME_FIELD,objInfo.getName());
+      rd.addField(GENERAL_DESCRIPTION_FIELD,objInfo.getComments());
+      if (creationDate != null)
+        rd.addField(GENERAL_CREATIONDATE_FIELD,new Long(creationDate.getTime()).toString());
+      if (modifyDate != null)
+        rd.addField(GENERAL_MODIFYDATE_FIELD,new Long(modifyDate.getTime()).toString());
+      if (parentID != null)
+        rd.addField(GENERAL_PARENTID,parentID.toString());
+      UserInformation owner = llc.getUserInformation(objInfo.getOwnerId().intValue());
+      UserInformation creator = llc.getUserInformation(objInfo.getCreatorId().intValue());
+      UserInformation modifier = llc.getUserInformation(versInfo.getOwnerId().intValue());
+      if (owner != null)
+        rd.addField(GENERAL_OWNER,owner.getName());
+      if (creator != null)
+        rd.addField(GENERAL_CREATOR,creator.getName());
+      if (modifier != null)
+        rd.addField(GENERAL_MODIFIER,modifier.getName());
+
+      // Iterate over the metadata items.  These are organized by category
+      // for speed of lookup.
+
+      Iterator<MetadataItem> catIter = desc.getItems(categoryPaths);
+      while (catIter.hasNext())
+      {
+        MetadataItem item = catIter.next();
+        MetadataPathItem pathItem = item.getPathItem();
+        if (pathItem != null)
+        {
+          int catID = pathItem.getCatID();
+          // grab the associated catversion
+          LLValue catVersion = getCatVersion(objID,catID);
+          if (catVersion != null)
+          {
+            // Go through attributes now
+            Iterator<String> attrIter = item.getAttributeNames();
+            while (attrIter.hasNext())
             {
-              MetadataItem item = catIter.next();
-              MetadataPathItem pathItem = item.getPathItem();
-              if (pathItem != null)
-              {
-                int catID = pathItem.getCatID();
-                // grab the associated catversion
-                LLValue catVersion = getCatVersion(objID,catID);
-                if (catVersion != null)
-                {
-                  // Go through attributes now
-                  Iterator<String> attrIter = item.getAttributeNames();
-                  while (attrIter.hasNext())
-                  {
-                    String attrName = attrIter.next();
-                    // Create a unique metadata name
-                    String metadataName = pathItem.getCatName()+":"+attrName;
-                    // Fetch the metadata and stuff it into the RepositoryData structure
-                    String[] metadataValue = getAttributeValue(catVersion,attrName);
-                    if (metadataValue != null)
-                      rd.addField(metadataName,metadataValue);
-                    else
-                      Logging.connectors.warn("Livelink: Metadata attribute '"+metadataName+"'
does not seem to exist; please correct the job");
-                  }
-                }
-
-              }
+              String attrName = attrIter.next();
+              // Create a unique metadata name
+              String metadataName = pathItem.getCatName()+":"+attrName;
+              // Fetch the metadata and stuff it into the RepositoryData structure
+              String[] metadataValue = getAttributeValue(catVersion,attrName);
+              if (metadataValue != null)
+                rd.addField(metadataName,metadataValue);
+              else
+                Logging.connectors.warn("Livelink: Metadata attribute '"+metadataName+"'
does not seem to exist; please correct the job");
             }
+          }
+          
+        }
+      }
 
-            if (actualAcls != null && denyAcls != null)
-              rd.setSecurity(RepositoryDocument.SECURITY_TYPE_DOCUMENT,actualAcls,denyAcls);
+      if (actualAcls != null && denyAcls != null)
+        rd.setSecurity(RepositoryDocument.SECURITY_TYPE_DOCUMENT,actualAcls,denyAcls);
 
-            // Add the path metadata item into the mix, if enabled
-            String pathAttributeName = sDesc.getPathAttributeName();
-            if (pathAttributeName != null && pathAttributeName.length() > 0)
-            {
-              String pathString = sDesc.getPathAttributeValue(documentIdentifier);
-              if (pathString != null)
-              {
-                if (Logging.connectors.isDebugEnabled())
-                  Logging.connectors.debug("Livelink: Path attribute name is '"+pathAttributeName+"'"+contextMsg+",
value is '"+pathString+"'");
-                rd.addField(pathAttributeName,pathString);
-              }
-            }
+      // Add the path metadata item into the mix, if enabled
+      String pathAttributeName = sDesc.getPathAttributeName();
+      if (pathAttributeName != null && pathAttributeName.length() > 0)
+      {
+        String pathString = sDesc.getPathAttributeValue(documentIdentifier);
+        if (pathString != null)
+        {
+          if (Logging.connectors.isDebugEnabled())
+            Logging.connectors.debug("Livelink: Path attribute name is '"+pathAttributeName+"'"+contextMsg+",
value is '"+pathString+"'");
+          rd.addField(pathAttributeName,pathString);
+        }
+      }
 
-            if (ingestProtocol != null)
-            {
-              // Use HTTP to fetch document!
-              String ingestHttpAddress = convertToIngestURI(documentIdentifier);
-              if (ingestHttpAddress != null)
-              {
+      if (ingestProtocol != null)
+      {
+        // Use HTTP to fetch document!
+        String ingestHttpAddress = convertToIngestURI(documentIdentifier);
+        if (ingestHttpAddress != null)
+        {
 
-                // Set up connection
-                HttpClient client = getInitializedClient(contextMsg);
+          // Set up connection
+          HttpClient client = getInitializedClient(contextMsg);
 
-                long currentTime;
+          long currentTime;
 
-                if (Logging.connectors.isInfoEnabled())
-                  Logging.connectors.info("Livelink: " + ingestHttpAddress);
+          if (Logging.connectors.isInfoEnabled())
+            Logging.connectors.info("Livelink: " + ingestHttpAddress);
 
 
-                HttpGet method = new HttpGet(getHost().toURI() + ingestHttpAddress);
-                method.setHeader(new BasicHeader("Accept","*/*"));
+          HttpGet method = new HttpGet(getHost().toURI() + ingestHttpAddress);
+          method.setHeader(new BasicHeader("Accept","*/*"));
 
-                ExecuteMethodThread methodThread = new ExecuteMethodThread(client,method);
-                methodThread.start();
+          ExecuteMethodThread methodThread = new ExecuteMethodThread(client,method);
+          methodThread.start();
+          try
+          {
+
+            int statusCode = methodThread.getResponseCode();
+            switch (statusCode)
+            {
+            case 500:
+            case 502:
+              Logging.connectors.warn("Livelink: Service interruption during fetch "+contextMsg+"
with Livelink HTTP Server, retrying...");
+              throw new ServiceInterruption("Service interruption during fetch",new ManifoldCFException(Integer.toString(statusCode)+"
error while fetching"),System.currentTimeMillis()+60000L,
+                System.currentTimeMillis()+600000L,-1,true);
+
+            case HttpStatus.SC_UNAUTHORIZED:
+              Logging.connectors.warn("Livelink: Document fetch unauthorized for "+ingestHttpAddress+"
("+contextMsg+")");
+              // Since we logged in, we should fail here if the ingestion user doesn't have
access to the
+              // the document, but if we do, don't fail hard.
+              resultCode = "UNAUTHORIZED";
+              activities.noDocument(documentIdentifier,version);
+              return;
+
+            case HttpStatus.SC_OK:
+              if (Logging.connectors.isDebugEnabled())
+                Logging.connectors.debug("Livelink: Created http document connection to Livelink
"+contextMsg);
+              // A non-existent content length will cause a value of -1 to be returned. 
This seems to indicate that the session login did not work right.
+              if (methodThread.getResponseContentLength() >= 0)
+              {
                 try
                 {
-
-                  int statusCode = methodThread.getResponseCode();
-                  switch (statusCode)
+                  InputStream is = methodThread.getSafeInputStream();
+                  try
                   {
-                  case 500:
-                  case 502:
-                    Logging.connectors.warn("Livelink: Service interruption during fetch
"+contextMsg+" with Livelink HTTP Server, retrying...");
-                    throw new ServiceInterruption("Service interruption during fetch",new
ManifoldCFException(Integer.toString(statusCode)+" error while fetching"),System.currentTimeMillis()+60000L,
-                      System.currentTimeMillis()+600000L,-1,true);
-
-                  case HttpStatus.SC_UNAUTHORIZED:
-                    Logging.connectors.warn("Livelink: Document fetch unauthorized for "+ingestHttpAddress+"
("+contextMsg+")");
-                    // Since we logged in, we should fail here if the ingestion user doesn't
have access to the
-                    // the document, but if we do, don't fail hard.
-                    resultCode = "UNAUTHORIZED";
-                    activities.noDocument(documentIdentifier,version);
-                    return;
-
-                  case HttpStatus.SC_OK:
-                    if (Logging.connectors.isDebugEnabled())
-                      Logging.connectors.debug("Livelink: Created http document connection
to Livelink "+contextMsg);
-                    // A non-existent content length will cause a value of -1 to be returned.
 This seems to indicate that the session login did not work right.
-                    if (methodThread.getResponseContentLength() >= 0)
-                    {
-                      try
-                      {
-                        InputStream is = methodThread.getSafeInputStream();
-                        try
-                        {
-                          rd.setBinary(is,dataSize);
+                    rd.setBinary(is,dataSize);
                             
-                          activities.ingestDocumentWithException(documentIdentifier,version,viewHttpAddress,rd);
-
-                          if (Logging.connectors.isDebugEnabled())
-                            Logging.connectors.debug("Livelink: Ingesting done "+contextMsg);
+                    activities.ingestDocumentWithException(documentIdentifier,version,viewHttpAddress,rd);
+                    
+                    if (Logging.connectors.isDebugEnabled())
+                      Logging.connectors.debug("Livelink: Ingesting done "+contextMsg);
 
-                        }
-                        finally
-                        {
-                          // Close stream via thread, since otherwise this can hang
-                          is.close();
-                        }
-                      }
-                      catch (java.net.SocketTimeoutException e)
-                      {
-                        resultCode = "DATATIMEOUT";
-                        resultDescription = e.getMessage();
-                        currentTime = System.currentTimeMillis();
-                        Logging.connectors.warn("Livelink: Livelink socket timed out ingesting
from the Livelink HTTP Server "+contextMsg+": "+e.getMessage(), e);
-                        throw new ServiceInterruption("Socket timed out: "+e.getMessage(),e,currentTime+300000L,currentTime+6*3600000L,-1,false);
-                      }
-                      catch (java.net.SocketException e)
-                      {
-                        resultCode = "DATASOCKETERROR";
-                        resultDescription = e.getMessage();
-                        currentTime = System.currentTimeMillis();
-                        Logging.connectors.warn("Livelink: Livelink socket error ingesting
from the Livelink HTTP Server "+contextMsg+": "+e.getMessage(), e);
-                        throw new ServiceInterruption("Socket error: "+e.getMessage(),e,currentTime+300000L,currentTime+6*3600000L,-1,false);
-                      }
-                      catch (javax.net.ssl.SSLHandshakeException e)
-                      {
-                        resultCode = "DATASSLHANDSHAKEERROR";
-                        resultDescription = e.getMessage();
-                        currentTime = System.currentTimeMillis();
-                        Logging.connectors.warn("Livelink: SSL handshake failed authenticating
"+contextMsg+": "+e.getMessage(),e);
-                        throw new ServiceInterruption("SSL handshake error: "+e.getMessage(),e,currentTime+60000L,currentTime+300000L,-1,true);
-                      }
-                      catch (ConnectTimeoutException e)
-                      {
-                        resultCode = "CONNECTTIMEOUT";
-                        resultDescription = e.getMessage();
-                        currentTime = System.currentTimeMillis();
-                        Logging.connectors.warn("Livelink: Livelink socket timed out connecting
to the Livelink HTTP Server "+contextMsg+": "+e.getMessage(), e);
-                        throw new ServiceInterruption("Connect timed out: "+e.getMessage(),e,currentTime+300000L,currentTime+6*3600000L,-1,false);
-                      }
-                      catch (InterruptedException e)
-                      {
-                        wasInterrupted = true;
-                        throw new ManifoldCFException("Interrupted: "+e.getMessage(),e,ManifoldCFException.INTERRUPTED);
-                      }
-                      catch (InterruptedIOException e)
-                      {
-                        wasInterrupted = true;
-                        throw new ManifoldCFException("Interrupted: "+e.getMessage(),e,ManifoldCFException.INTERRUPTED);
-                      }
-                      catch (HttpException e)
-                      {
-                        resultCode = "HTTPEXCEPTION";
-                        resultDescription = e.getMessage();
-                        // Treat unknown error ingesting data as a transient condition
-                        currentTime = System.currentTimeMillis();
-                        Logging.connectors.warn("Livelink: HTTP exception ingesting "+contextMsg+":
"+e.getMessage(),e);
-                        throw new ServiceInterruption("HTTP exception ingesting "+contextMsg+":
"+e.getMessage(),e,currentTime+300000L,currentTime+6*3600000L,-1,false);
-                      }
-                      catch (IOException e)
-                      {
-                        resultCode = "DATAEXCEPTION";
-                        resultDescription = e.getMessage();
-                        // Treat unknown error ingesting data as a transient condition
-                        currentTime = System.currentTimeMillis();
-                        Logging.connectors.warn("Livelink: IO exception ingesting "+contextMsg+":
"+e.getMessage(),e);
-                        throw new ServiceInterruption("IO exception ingesting "+contextMsg+":
"+e.getMessage(),e,currentTime+300000L,currentTime+6*3600000L,-1,false);
-                      }
-                      readSize = dataSize;
-                    }
-                    else
-                    {
-                      resultCode = "SESSIONLOGINFAILED";
-                      activities.noDocument(documentIdentifier,version);
-                    }
-                    break;
-                  case HttpStatus.SC_BAD_REQUEST:
-                  case HttpStatus.SC_USE_PROXY:
-                  case HttpStatus.SC_GONE:
-                    resultCode = "ERROR "+Integer.toString(statusCode);
-                    throw new ManifoldCFException("Unrecoverable request failure; error =
"+Integer.toString(statusCode));
-                  default:
-                    resultCode = "UNKNOWN";
-                    Logging.connectors.warn("Livelink: Attempt to retrieve document from
'"+ingestHttpAddress+"' received a response of "+Integer.toString(statusCode)+"; retrying
in one minute");
-                    currentTime = System.currentTimeMillis();
-                    throw new ServiceInterruption("Fetch failed; retrying in 1 minute",new
ManifoldCFException("Fetch failed with unknown code "+Integer.toString(statusCode)),
-                      currentTime+60000L,currentTime+600000L,-1,true);
                   }
-                }
-                catch (InterruptedException e)
-                {
-                  // Drop the connection on the floor
-                  methodThread.interrupt();
-                  methodThread = null;
-                  throw new ManifoldCFException("Interrupted: "+e.getMessage(),e,ManifoldCFException.INTERRUPTED);
+                  finally
+                  {
+                    // Close stream via thread, since otherwise this can hang
+                    is.close();
+                  }
                 }
                 catch (java.net.SocketTimeoutException e)
                 {
-                  Logging.connectors.warn("Livelink: Socket timed out reading from the Livelink
HTTP Server "+contextMsg+": "+e.getMessage(), e);
-                  resultCode = "TIMEOUT";
+                  resultCode = "DATATIMEOUT";
                   resultDescription = e.getMessage();
                   currentTime = System.currentTimeMillis();
-                  throw new ServiceInterruption("Socket timed out: "+e.getMessage(),e,currentTime+300000L,currentTime+6*3600000L,-1,true);
+                  Logging.connectors.warn("Livelink: Livelink socket timed out ingesting
from the Livelink HTTP Server "+contextMsg+": "+e.getMessage(), e);
+                  throw new ServiceInterruption("Socket timed out: "+e.getMessage(),e,currentTime+300000L,currentTime+6*3600000L,-1,false);
                 }
                 catch (java.net.SocketException e)
                 {
-                  Logging.connectors.warn("Livelink: Socket error reading from Livelink HTTP
Server "+contextMsg+": "+e.getMessage(), e);
-                  resultCode = "SOCKETERROR";
+                  resultCode = "DATASOCKETERROR";
                   resultDescription = e.getMessage();
                   currentTime = System.currentTimeMillis();
-                  throw new ServiceInterruption("Socket error: "+e.getMessage(),e,currentTime+300000L,currentTime+6*3600000L,-1,true);
+                  Logging.connectors.warn("Livelink: Livelink socket error ingesting from
the Livelink HTTP Server "+contextMsg+": "+e.getMessage(), e);
+                  throw new ServiceInterruption("Socket error: "+e.getMessage(),e,currentTime+300000L,currentTime+6*3600000L,-1,false);
                 }
                 catch (javax.net.ssl.SSLHandshakeException e)
                 {
-                  currentTime = System.currentTimeMillis();
-                  Logging.connectors.warn("Livelink: SSL handshake failed "+contextMsg+":
"+e.getMessage(),e);
-                  resultCode = "SSLHANDSHAKEERROR";
+                  resultCode = "DATASSLHANDSHAKEERROR";
                   resultDescription = e.getMessage();
+                  currentTime = System.currentTimeMillis();
+                  Logging.connectors.warn("Livelink: SSL handshake failed authenticating
"+contextMsg+": "+e.getMessage(),e);
                   throw new ServiceInterruption("SSL handshake error: "+e.getMessage(),e,currentTime+60000L,currentTime+300000L,-1,true);
                 }
                 catch (ConnectTimeoutException e)
                 {
-                  Logging.connectors.warn("Livelink: Connect timed out reading from the Livelink
HTTP Server "+contextMsg+": "+e.getMessage(), e);
                   resultCode = "CONNECTTIMEOUT";
                   resultDescription = e.getMessage();
                   currentTime = System.currentTimeMillis();
-                  throw new ServiceInterruption("Connect timed out: "+e.getMessage(),e,currentTime+300000L,currentTime+6*3600000L,-1,true);
+                  Logging.connectors.warn("Livelink: Livelink socket timed out connecting
to the Livelink HTTP Server "+contextMsg+": "+e.getMessage(), e);
+                  throw new ServiceInterruption("Connect timed out: "+e.getMessage(),e,currentTime+300000L,currentTime+6*3600000L,-1,false);
+                }
+                catch (InterruptedException e)
+                {
+                  wasInterrupted = true;
+                  throw new ManifoldCFException("Interrupted: "+e.getMessage(),e,ManifoldCFException.INTERRUPTED);
                 }
                 catch (InterruptedIOException e)
                 {
-                  methodThread.interrupt();
+                  wasInterrupted = true;
                   throw new ManifoldCFException("Interrupted: "+e.getMessage(),e,ManifoldCFException.INTERRUPTED);
                 }
                 catch (HttpException e)
                 {
-                  resultCode = "EXCEPTION";
+                  resultCode = "HTTPEXCEPTION";
                   resultDescription = e.getMessage();
-                  throw new ManifoldCFException("Exception getting response "+contextMsg+":
"+e.getMessage(), e);
+                  // Treat unknown error ingesting data as a transient condition
+                  currentTime = System.currentTimeMillis();
+                  Logging.connectors.warn("Livelink: HTTP exception ingesting "+contextMsg+":
"+e.getMessage(),e);
+                  throw new ServiceInterruption("HTTP exception ingesting "+contextMsg+":
"+e.getMessage(),e,currentTime+300000L,currentTime+6*3600000L,-1,false);
                 }
                 catch (IOException e)
                 {
-                  resultCode = "EXCEPTION";
+                  resultCode = "DATAEXCEPTION";
                   resultDescription = e.getMessage();
-                  throw new ManifoldCFException("Exception getting response "+contextMsg+":
"+e.getMessage(), e);
-                }
-                finally
-                {
-                  if (methodThread != null)
-                  {
-                    methodThread.abort();
-                    if (!wasInterrupted)
-                    {
-                      try
-                      {
-                       methodThread.finishUp();
-                      }
-                      catch (InterruptedException e)
-                      {
-                        wasInterrupted = true;
-                        throw new ManifoldCFException(e.getMessage(),e,ManifoldCFException.INTERRUPTED);
-                      }
-                    }
-                  }
+                  // Treat unknown error ingesting data as a transient condition
+                  currentTime = System.currentTimeMillis();
+                  Logging.connectors.warn("Livelink: IO exception ingesting "+contextMsg+":
"+e.getMessage(),e);
+                  throw new ServiceInterruption("IO exception ingesting "+contextMsg+": "+e.getMessage(),e,currentTime+300000L,currentTime+6*3600000L,-1,false);
                 }
+                readSize = dataSize;
               }
               else
               {
-                if (Logging.connectors.isDebugEnabled())
-                  Logging.connectors.debug("Livelink: No fetch URI "+contextMsg+" - not ingesting");
-                resultCode = "NOURI";
-                return;
+                resultCode = "SESSIONLOGINFAILED";
+                activities.noDocument(documentIdentifier,version);
               }
+              break;
+            case HttpStatus.SC_BAD_REQUEST:
+            case HttpStatus.SC_USE_PROXY:
+            case HttpStatus.SC_GONE:
+              resultCode = "ERROR "+Integer.toString(statusCode);
+              throw new ManifoldCFException("Unrecoverable request failure; error = "+Integer.toString(statusCode));
+            default:
+              resultCode = "UNKNOWN";
+              Logging.connectors.warn("Livelink: Attempt to retrieve document from '"+ingestHttpAddress+"'
received a response of "+Integer.toString(statusCode)+"; retrying in one minute");
+              currentTime = System.currentTimeMillis();
+              throw new ServiceInterruption("Fetch failed; retrying in 1 minute",new ManifoldCFException("Fetch
failed with unknown code "+Integer.toString(statusCode)),
+                currentTime+60000L,currentTime+600000L,-1,true);
             }
-            else
+          }
+          catch (InterruptedException e)
+          {
+            // Drop the connection on the floor
+            methodThread.interrupt();
+            methodThread = null;
+            throw new ManifoldCFException("Interrupted: "+e.getMessage(),e,ManifoldCFException.INTERRUPTED);
+          }
+          catch (java.net.SocketTimeoutException e)
+          {
+            Logging.connectors.warn("Livelink: Socket timed out reading from the Livelink
HTTP Server "+contextMsg+": "+e.getMessage(), e);
+            resultCode = "TIMEOUT";
+            resultDescription = e.getMessage();
+            currentTime = System.currentTimeMillis();
+            throw new ServiceInterruption("Socket timed out: "+e.getMessage(),e,currentTime+300000L,currentTime+6*3600000L,-1,true);
+          }
+          catch (java.net.SocketException e)
+          {
+            Logging.connectors.warn("Livelink: Socket error reading from Livelink HTTP Server
"+contextMsg+": "+e.getMessage(), e);
+            resultCode = "SOCKETERROR";
+            resultDescription = e.getMessage();
+            currentTime = System.currentTimeMillis();
+            throw new ServiceInterruption("Socket error: "+e.getMessage(),e,currentTime+300000L,currentTime+6*3600000L,-1,true);
+          }
+          catch (javax.net.ssl.SSLHandshakeException e)
+          {
+            currentTime = System.currentTimeMillis();
+            Logging.connectors.warn("Livelink: SSL handshake failed "+contextMsg+": "+e.getMessage(),e);
+            resultCode = "SSLHANDSHAKEERROR";
+            resultDescription = e.getMessage();
+            throw new ServiceInterruption("SSL handshake error: "+e.getMessage(),e,currentTime+60000L,currentTime+300000L,-1,true);
+          }
+          catch (ConnectTimeoutException e)
+          {
+            Logging.connectors.warn("Livelink: Connect timed out reading from the Livelink
HTTP Server "+contextMsg+": "+e.getMessage(), e);
+            resultCode = "CONNECTTIMEOUT";
+            resultDescription = e.getMessage();
+            currentTime = System.currentTimeMillis();
+            throw new ServiceInterruption("Connect timed out: "+e.getMessage(),e,currentTime+300000L,currentTime+6*3600000L,-1,true);
+          }
+          catch (InterruptedIOException e)
+          {
+            methodThread.interrupt();
+            throw new ManifoldCFException("Interrupted: "+e.getMessage(),e,ManifoldCFException.INTERRUPTED);
+          }
+          catch (HttpException e)
+          {
+            resultCode = "EXCEPTION";
+            resultDescription = e.getMessage();
+            throw new ManifoldCFException("Exception getting response "+contextMsg+": "+e.getMessage(),
e);
+          }
+          catch (IOException e)
+          {
+            resultCode = "EXCEPTION";
+            resultDescription = e.getMessage();
+            throw new ManifoldCFException("Exception getting response "+contextMsg+": "+e.getMessage(),
e);
+          }
+          finally
+          {
+            if (methodThread != null)
             {
-              // Use FetchVersion instead
-              long currentTime;
-              
-              // Fire up the document reading thread
-              DocumentReadingThread t = new DocumentReadingThread(vol,objID,0);
-              try 
+              methodThread.abort();
+              if (!wasInterrupted)
               {
-                t.start();
                 try
                 {
-                  InputStream is = t.getSafeInputStream();
-                  try 
-                  {
-                    // Can only index while background thread is running!
-                    rd.setBinary(is, dataSize);
-                    activities.ingestDocumentWithException(documentIdentifier, version, viewHttpAddress,
rd);
-                  }
-                  finally
-                  {
-                    is.close();
-                  }
-                }
-                catch (ManifoldCFException e)
-                {
-                  if (e.getErrorCode() == ManifoldCFException.INTERRUPTED)
-                    wasInterrupted = true;
-                  throw e;
+                  methodThread.finishUp();
                 }
-                catch (java.net.SocketTimeoutException e)
-                {
-                  throw e;
-                }
-                catch (InterruptedIOException e)
+                catch (InterruptedException e)
                 {
                   wasInterrupted = true;
-                  throw e;
-                }
-                finally
-                {
-                  if (!wasInterrupted)
-                    t.finishUp();
-                }
-
-                // No errors.  Record the fact that we made it.
-                resultCode = "OK";
-                readSize = dataSize;
-              }
-              catch (InterruptedException e) 
-              {
-                t.interrupt();
-                throw new ManifoldCFException("Interrupted: " + e.getMessage(), e,
-                  ManifoldCFException.INTERRUPTED);
-              }
-              catch (ConnectTimeoutException e)
-              {
-                Logging.connectors.warn("Livelink: Connect timed out "+contextMsg+": "+e.getMessage(),
e);
-                resultCode = "CONNECTTIMEOUT";
-                resultDescription = e.getMessage();
-                currentTime = System.currentTimeMillis();
-                throw new ServiceInterruption("Connect timed out: "+e.getMessage(),e,currentTime+300000L,currentTime+6*3600000L,-1,true);
-              }
-              catch (InterruptedIOException e)
-              {
-                t.interrupt();
-                throw new ManifoldCFException("Interrupted: "+e.getMessage(),e,ManifoldCFException.INTERRUPTED);
-              }
-              catch (IOException e)
-              {
-                resultCode = "EXCEPTION";
-                resultDescription = e.getMessage();
-                throw new ManifoldCFException("Exception getting response "+contextMsg+":
"+e.getMessage(), e);
-              }
-              catch (ManifoldCFException e)
-              {
-                if (e.getErrorCode() != ManifoldCFException.INTERRUPTED)
-                {
-                  resultCode = "EXCEPTION";
-                  resultDescription = e.getMessage();
+                  throw new ManifoldCFException(e.getMessage(),e,ManifoldCFException.INTERRUPTED);
                 }
-                throw e;
-              }
-              catch (RuntimeException e)
-              {
-                resultCode = "EXCEPTION";
-                resultDescription = e.getMessage();
-                handleLivelinkRuntimeException(e,0,true);
               }
             }
           }
-          else
-          {
-            // Document not indexable because of its length
-            resultDescription = "Document length ("+dataSize+") was rejected by output connector";
-            if (Logging.connectors.isDebugEnabled())
-              Logging.connectors.debug("Livelink: Excluding document "+documentIdentifier+"
because its length ("+dataSize+") was rejected by output connector");
-            resultCode = "DOCUMENTTOOLONG";
-            activities.noDocument(documentIdentifier,version);
-          }
         }
         else
         {
-          // Document not indexable because of its mime type
-          resultDescription = "Mime type ("+mimeType+") was rejected by output connector";
           if (Logging.connectors.isDebugEnabled())
-            Logging.connectors.debug("Livelink: Excluding document "+documentIdentifier+"
because its mime type ("+mimeType+") was rejected by output connector");
-          resultCode = "MIMETYPEEXCLUSION";
-          activities.noDocument(documentIdentifier,version);
+            Logging.connectors.debug("Livelink: No fetch URI "+contextMsg+" - not ingesting");
+          resultCode = "NOURI";
+          return;
         }
       }
       else
       {
-        // Document not ingestable due to URL
-        resultDescription = "URL ("+viewHttpAddress+") was rejected by output connector";
-        if (Logging.connectors.isDebugEnabled())
-          Logging.connectors.debug("Livelink: Excluding document "+documentIdentifier+" because
its URL ("+viewHttpAddress+") was rejected by output connector");
-        resultCode = "URLEXCLUSION";
-        activities.noDocument(documentIdentifier,version);
+        // Use FetchVersion instead
+        long currentTime;
+              
+        // Fire up the document reading thread
+        DocumentReadingThread t = new DocumentReadingThread(vol,objID,0);
+        try 
+        {
+          t.start();
+          try
+          {
+            InputStream is = t.getSafeInputStream();
+            try 
+            {
+              // Can only index while background thread is running!
+              rd.setBinary(is, dataSize);
+              activities.ingestDocumentWithException(documentIdentifier, version, viewHttpAddress,
rd);
+            }
+            finally
+            {
+              is.close();
+            }
+          }
+          catch (ManifoldCFException e)
+          {
+            if (e.getErrorCode() == ManifoldCFException.INTERRUPTED)
+              wasInterrupted = true;
+            throw e;
+          }
+          catch (java.net.SocketTimeoutException e)
+          {
+            throw e;
+          }
+          catch (InterruptedIOException e)
+          {
+            wasInterrupted = true;
+            throw e;
+          }
+          finally
+          {
+            if (!wasInterrupted)
+              t.finishUp();
+          }
+
+          // No errors.  Record the fact that we made it.
+          resultCode = "OK";
+          readSize = dataSize;
+        }
+        catch (InterruptedException e) 
+        {
+          t.interrupt();
+          throw new ManifoldCFException("Interrupted: " + e.getMessage(), e,
+            ManifoldCFException.INTERRUPTED);
+        }
+        catch (ConnectTimeoutException e)
+        {
+          Logging.connectors.warn("Livelink: Connect timed out "+contextMsg+": "+e.getMessage(),
e);
+          resultCode = "CONNECTTIMEOUT";
+          resultDescription = e.getMessage();
+          currentTime = System.currentTimeMillis();
+          throw new ServiceInterruption("Connect timed out: "+e.getMessage(),e,currentTime+300000L,currentTime+6*3600000L,-1,true);
+        }
+        catch (InterruptedIOException e)
+        {
+          t.interrupt();
+          throw new ManifoldCFException("Interrupted: "+e.getMessage(),e,ManifoldCFException.INTERRUPTED);
+        }
+        catch (IOException e)
+        {
+          resultCode = "EXCEPTION";
+          resultDescription = e.getMessage();
+          throw new ManifoldCFException("Exception getting response "+contextMsg+": "+e.getMessage(),
e);
+        }
+        catch (ManifoldCFException e)
+        {
+          if (e.getErrorCode() != ManifoldCFException.INTERRUPTED)
+          {
+            resultCode = "EXCEPTION";
+            resultDescription = e.getMessage();
+          }
+          throw e;
+        }
+        catch (RuntimeException e)
+        {
+          resultCode = "EXCEPTION";
+          resultDescription = e.getMessage();
+          handleLivelinkRuntimeException(e,0,true);
+        }
       }
     }
     finally



Mime
View raw message