manifoldcf-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kwri...@apache.org
Subject svn commit: r1633326 - in /manifoldcf/branches/dev_1x: ./ connectors/filesystem/connector/src/main/java/org/apache/manifoldcf/agents/output/filesystem/ connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/ connectors/solr/co...
Date Tue, 21 Oct 2014 09:32:07 GMT
Author: kwright
Date: Tue Oct 21 09:32:07 2014
New Revision: 1633326

URL: http://svn.apache.org/r1633326
Log:
Pull up more CONNECTORS-1077 changes from trunk

Modified:
    manifoldcf/branches/dev_1x/   (props changed)
    manifoldcf/branches/dev_1x/connectors/filesystem/connector/src/main/java/org/apache/manifoldcf/agents/output/filesystem/FileOutputConnector.java
    manifoldcf/branches/dev_1x/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputConnector.java
    manifoldcf/branches/dev_1x/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java

Propchange: manifoldcf/branches/dev_1x/
------------------------------------------------------------------------------
  Merged /manifoldcf/trunk:r1633282,1633284,1633295

Modified: manifoldcf/branches/dev_1x/connectors/filesystem/connector/src/main/java/org/apache/manifoldcf/agents/output/filesystem/FileOutputConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/connectors/filesystem/connector/src/main/java/org/apache/manifoldcf/agents/output/filesystem/FileOutputConnector.java?rev=1633326&r1=1633325&r2=1633326&view=diff
==============================================================================
--- manifoldcf/branches/dev_1x/connectors/filesystem/connector/src/main/java/org/apache/manifoldcf/agents/output/filesystem/FileOutputConnector.java
(original)
+++ manifoldcf/branches/dev_1x/connectors/filesystem/connector/src/main/java/org/apache/manifoldcf/agents/output/filesystem/FileOutputConnector.java
Tue Oct 21 09:32:07 2014
@@ -169,6 +169,9 @@ public class FileOutputConnector extends
     FileOutputSpecs specs = new FileOutputSpecs(getSpecNode(outputDescription.getSpecification()));;
     StringBuffer path = new StringBuffer();
 
+    String errorCode = "OK";
+    String errorDesc = null;
+
     try {
       /*
         * make file path
@@ -215,8 +218,9 @@ public class FileOutputConnector extends
           }
           else
           {
-              activities.recordActivity(null,INGEST_ACTIVITY,null,documentURI,activities.CREATED_DIRECTORY,"Couldn't
create directory. '"+newPath+"'");
-              throw new ManifoldCFException("Could not create directory '"+newPath+"'.  Permission
issue?");
+              errorCode = activities.CREATED_DIRECTORY;
+              errorDesc = "Could not create directory '\"+newPath+\"'.  Permission issue?";
+              throw new ManifoldCFException(errorDesc);
           }
         }
         // Directory successfully created!
@@ -247,8 +251,9 @@ public class FileOutputConnector extends
             continue;
           }
           // Probably some other error
-          activities.recordActivity(null, INGEST_ACTIVITY, new Long(document.getBinaryLength()),
documentURI, activities.EXCEPTION, "Unexpected exception while creating the file.'" + outputPath
+ "'");
-          throw new ManifoldCFException("Could not create file '"+outputPath+"': "+e.getMessage(),e);
+          errorCode = e.getClass().getSimpleName().toUpperCase(Locale.ROOT);
+          errorDesc = "Could not create file '" + outputPath + "': " +e.getMessage();
+          throw new ManifoldCFException(errorDesc,e);
         }
       }
 
@@ -259,8 +264,9 @@ public class FileOutputConnector extends
         FileChannel channel = output.getChannel();
         FileLock lock = channel.tryLock();
         if (lock == null){
-          activities.recordActivity(null,INGEST_ACTIVITY, new Long(document.getBinaryLength()),
documentURI,activities.EXCEPTION,"Could not lock file: '" + outputPath + "'");
-          throw new ServiceInterruption("Could not lock file: '"+outputPath+"'",null,1000L,-1L,10,false);
+          errorCode = ServiceInterruption.class.getSimpleName().toUpperCase(Locale.ROOT);
+          errorDesc = "Could not lock file: '"+outputPath+"'";
+          throw new ServiceInterruption(errorDesc,null,1000L,-1L,10,false);
         }
 
 
@@ -292,24 +298,29 @@ public class FileOutputConnector extends
         }
       }
     } catch (URISyntaxException e) {
-      activities.recordActivity(null,INGEST_ACTIVITY,new Long(document.getBinaryLength()),
documentURI,activities.EXCEPTION,"Failed to write document due to: " + e.getMessage());
+      errorCode = e.getClass().getSimpleName().toUpperCase(Locale.ROOT);
+      errorDesc = "Failed to write document due to: " + e.getMessage();
       handleURISyntaxException(e);
       return DOCUMENTSTATUS_REJECTED;
     } catch (FileNotFoundException e) {
-      activities.recordActivity(null,INGEST_ACTIVITY,new Long(document.getBinaryLength()),
documentURI,activities.EXCEPTION,"Failed to write document due to: " + e.getMessage());
+      errorCode = e.getClass().getSimpleName().toUpperCase(Locale.ROOT);
+      errorDesc = "Failed to write document due to: " + e.getMessage();
       handleFileNotFoundException(e);
       return DOCUMENTSTATUS_REJECTED;
     } catch (SecurityException e) {
-      activities.recordActivity(null,INGEST_ACTIVITY,new Long(document.getBinaryLength()),
documentURI,activities.EXCEPTION,"Failed to write document due to: " + e.getMessage());
+      errorCode = e.getClass().getSimpleName().toUpperCase(Locale.ROOT);
+      errorDesc = "Failed to write document due to: " + e.getMessage();
       handleSecurityException(e);
       return DOCUMENTSTATUS_REJECTED;
     } catch (IOException e) {
-      activities.recordActivity(null,INGEST_ACTIVITY,new Long(document.getBinaryLength()),
documentURI ,activities.EXCEPTION,"Failed to write document due to: " + e.getMessage());
+      errorCode = e.getClass().getSimpleName().toUpperCase(Locale.ROOT);
+      errorDesc = "Failed to write document due to: " + e.getMessage();
       handleIOException(e);
       return DOCUMENTSTATUS_REJECTED;
+    } finally {
+        activities.recordActivity(null, INGEST_ACTIVITY, new Long(document.getBinaryLength()),
documentURI, errorCode, errorDesc);
     }
 
-    activities.recordActivity(null, INGEST_ACTIVITY, new Long(document.getBinaryLength()),
documentURI, "OK", null);
     return DOCUMENTSTATUS_ACCEPTED;
   }
 
@@ -361,6 +372,9 @@ public class FileOutputConnector extends
     // Establish a session
     getSession();
 
+    String errorCode = "OK";
+    String errorDesc = null;
+
     FileOutputConfig config = getConfigParameters(null);
 
     StringBuffer path = new StringBuffer();
@@ -432,27 +446,32 @@ public class FileOutputConnector extends
         catch (FileNotFoundException e)
         {
           // Probably some other error
-          activities.recordActivity(null,REMOVE_ACTIVITY,null,documentURI,activities.EXCEPTION,"Couldn't
delete the file due to FileNotFoundException '"+outputPath+"':" + e.getMessage());
+          errorCode = e.getClass().getSimpleName().toUpperCase(Locale.ROOT);
+          errorDesc = "Couldn't delete the file due to:" + e.getMessage();
           throw new ManifoldCFException("Could not zero out file '"+outputPath+"': "+e.getMessage(),e);
         }
       }
       // Just close it, to make a zero-length grave marker.
       output.close();
     } catch (URISyntaxException e) {
-      activities.recordActivity(null,REMOVE_ACTIVITY,null, documentURI,activities.EXCEPTION,"Failed
to delete document due to: " + e.getMessage());
+      errorCode = e.getClass().getSimpleName().toUpperCase(Locale.ROOT);
+      errorDesc = "Failed to delete document due to: " + e.getMessage();
       handleURISyntaxException(e);
     } catch (FileNotFoundException e) {
-      activities.recordActivity(null,REMOVE_ACTIVITY,null, documentURI,activities.EXCEPTION,"Failed
to delete document due to: " + e.getMessage());
+      errorCode = e.getClass().getSimpleName().toUpperCase(Locale.ROOT);
+      errorDesc = "Failed to delete document due to: " + e.getMessage();
       handleFileNotFoundException(e);
     } catch (SecurityException e) {
-      activities.recordActivity(null,REMOVE_ACTIVITY,null, documentURI,activities.EXCEPTION,"Failed
to delete document due to: " + e.getMessage());
+      errorCode = e.getClass().getSimpleName().toUpperCase(Locale.ROOT);
+      errorDesc = "Failed to delete document due to: " + e.getMessage();
       handleSecurityException(e);
     } catch (IOException e) {
-      activities.recordActivity(null,REMOVE_ACTIVITY,null, documentURI ,activities.EXCEPTION,"Failed
to delete document due to: " + e.getMessage());
+      errorCode = e.getClass().getSimpleName().toUpperCase(Locale.ROOT);
+      errorDesc = "Failed to delete document due to: " + e.getMessage();
       handleIOException(e);
+    } finally {
+        activities.recordActivity(null, REMOVE_ACTIVITY, null, documentURI, errorCode, errorDesc);
     }
-
-    activities.recordActivity(null, REMOVE_ACTIVITY, null, documentURI, "OK", null);
   }
 
   /** Obtain the name of the form check javascript method to call.

Modified: manifoldcf/branches/dev_1x/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputConnector.java?rev=1633326&r1=1633325&r2=1633326&view=diff
==============================================================================
--- manifoldcf/branches/dev_1x/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputConnector.java
(original)
+++ manifoldcf/branches/dev_1x/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputConnector.java
Tue Oct 21 09:32:07 2014
@@ -290,7 +290,7 @@ public class HDFSOutputConnector extends
       activities.recordActivity(startTime, INGEST_ACTIVITY, new Long(document.getBinaryLength()),
documentURI, "OK", null);
       return DOCUMENTSTATUS_ACCEPTED;
     } catch (URISyntaxException e) {
-      activities.recordActivity(null,INGEST_ACTIVITY,new Long(document.getBinaryLength()),documentURI,activities.EXCEPTION,"Failed
to write document due to: " + e.getMessage());
+      activities.recordActivity(null,INGEST_ACTIVITY,new Long(document.getBinaryLength()),documentURI,e.getClass().getSimpleName().toUpperCase(Locale.ROOT),"Failed
to write document due to: " + e.getMessage());
       handleURISyntaxException(e);
       return DOCUMENTSTATUS_REJECTED;
     }
@@ -324,10 +324,10 @@ public class HDFSOutputConnector extends
       deleteFile(path,activities,documentURI);
       activities.recordActivity(startTime, REMOVE_ACTIVITY, null, documentURI, "OK", null);
     } catch (JSONException e) {
-      activities.recordActivity(null,REMOVE_ACTIVITY,null,documentURI,activities.EXCEPTION,"Failed
to delete document due to: " + e.getMessage());
+      activities.recordActivity(null,REMOVE_ACTIVITY,null,documentURI,e.getClass().getSimpleName().toUpperCase(Locale.ROOT),"Failed
to delete document due to: " + e.getMessage());
       handleJSONException(e);
     } catch (URISyntaxException e) {
-      activities.recordActivity(null,REMOVE_ACTIVITY,null,documentURI,activities.EXCEPTION,"Failed
to delete document due to: " + e.getMessage());
+      activities.recordActivity(null,REMOVE_ACTIVITY,null,documentURI,e.getClass().getSimpleName().toUpperCase(Locale.ROOT),"Failed
to delete document due to: " + e.getMessage());
       handleURISyntaxException(e);
     }
   }
@@ -675,23 +675,33 @@ public class HDFSOutputConnector extends
   protected void createFile(Path path, InputStream input,IOutputAddActivity activities, String
documentURI)
     throws ManifoldCFException, ServiceInterruption {
     CreateFileThread t = new CreateFileThread(getSession(), path, input);
+    String errorCode = null;
+    String errorDesc = null;
     try {
       t.start();
       t.finishUp();
     } catch (InterruptedException e) {
       t.interrupt();
-      activities.recordActivity(null,INGEST_ACTIVITY,null,documentURI,activities.EXCEPTION,"Failed
to write document due to: " + e.getMessage());
+      errorCode = e.getClass().getSimpleName().toUpperCase(Locale.ROOT);
+      errorDesc = "Failed to write document due to: " + e.getMessage();
       throw new ManifoldCFException("Interrupted: "+e.getMessage(),e,ManifoldCFException.INTERRUPTED);
     } catch (java.net.SocketTimeoutException e) {
-      activities.recordActivity(null,INGEST_ACTIVITY,null,documentURI,activities.EXCEPTION,"Failed
to write document due to: " + e.getMessage());
+      errorCode = e.getClass().getSimpleName().toUpperCase(Locale.ROOT);
+      errorDesc = "Failed to write document due to: " + e.getMessage();
       handleIOException(e);
     } catch (InterruptedIOException e) {
       t.interrupt();
-      activities.recordActivity(null,INGEST_ACTIVITY,null,documentURI,activities.EXCEPTION,"Failed
to write document due to: " + e.getMessage());
+      errorCode = e.getClass().getSimpleName().toUpperCase(Locale.ROOT);
+      errorDesc = "Failed to write document due to: " + e.getMessage();
       handleIOException(e);
     } catch (IOException e) {
-      activities.recordActivity(null,INGEST_ACTIVITY,null,documentURI,activities.EXCEPTION,"Failed
to write document due to: " + e.getMessage());
+      errorCode = e.getClass().getSimpleName().toUpperCase(Locale.ROOT);
+      errorDesc = "Failed to write document due to: " + e.getMessage();
       handleIOException(e);
+    } finally {
+      if(errorCode != null & errorDesc != null){
+        activities.recordActivity(null,INGEST_ACTIVITY,null,documentURI,errorCode,errorDesc);
+      }
     }
   }
 
@@ -734,23 +744,33 @@ public class HDFSOutputConnector extends
     throws ManifoldCFException, ServiceInterruption {
     // Establish a session
     DeleteFileThread t = new DeleteFileThread(getSession(),path);
+    String errorCode = null;
+    String errorDesc = null;
     try {
       t.start();
       t.finishUp();
     } catch (InterruptedException e) {
-      t.interrupt();
-      activities.recordActivity(null,REMOVE_ACTIVITY,null,documentURI,activities.EXCEPTION,"Failed
to delete document due to: " + e.getMessage());
-      throw new ManifoldCFException("Interrupted: "+e.getMessage(),e,ManifoldCFException.INTERRUPTED);
+        t.interrupt();
+        errorCode = e.getClass().getSimpleName().toUpperCase(Locale.ROOT);
+        errorDesc = "Failed to write document due to: " + e.getMessage();
+        throw new ManifoldCFException("Interrupted: "+e.getMessage(),e,ManifoldCFException.INTERRUPTED);
     } catch (java.net.SocketTimeoutException e) {
-      activities.recordActivity(null,REMOVE_ACTIVITY,null,documentURI,activities.EXCEPTION,"Failed
to delete document due to: " + e.getMessage());
-      handleIOException(e);
+        errorCode = e.getClass().getSimpleName().toUpperCase(Locale.ROOT);
+        errorDesc = "Failed to write document due to: " + e.getMessage();
+        handleIOException(e);
     } catch (InterruptedIOException e) {
-      t.interrupt();
-      activities.recordActivity(null,REMOVE_ACTIVITY,null,documentURI,activities.EXCEPTION,"Failed
to delete document due to: " + e.getMessage());
-      handleIOException(e);
+        t.interrupt();
+        errorCode = e.getClass().getSimpleName().toUpperCase(Locale.ROOT);
+        errorDesc = "Failed to write document due to: " + e.getMessage();
+        handleIOException(e);
     } catch (IOException e) {
-      activities.recordActivity(null,REMOVE_ACTIVITY,null,documentURI,activities.EXCEPTION,"Failed
to delete document due to: " + e.getMessage());
-      handleIOException(e);
+        errorCode = e.getClass().getSimpleName().toUpperCase(Locale.ROOT);
+        errorDesc = "Failed to write document due to: " + e.getMessage();
+        handleIOException(e);
+    } finally {
+        if(errorCode != null & errorDesc != null){
+            activities.recordActivity(null,REMOVE_ACTIVITY,null,documentURI,errorCode,errorDesc);
+        }
     }
   }
 

Modified: manifoldcf/branches/dev_1x/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java?rev=1633326&r1=1633325&r2=1633326&view=diff
==============================================================================
--- manifoldcf/branches/dev_1x/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java
(original)
+++ manifoldcf/branches/dev_1x/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java
Tue Oct 21 09:32:07 2014
@@ -536,8 +536,11 @@ public class HttpPoster
       Logging.ingest.debug("indexPost(): '" + documentURI + "'");
 
     // If the document is too long, reject it.
-    if (maxDocumentLength != null && document.getBinaryLength() > maxDocumentLength.longValue())
+    if (maxDocumentLength != null && document.getBinaryLength() > maxDocumentLength.longValue()){
+      activities.recordActivity(null,SolrConnector.INGEST_ACTIVITY,null,documentURI,activities.EXCLUDED_LENGTH,"Solr
connector rejected document due to its big size. ('"+document.getBinaryLength()+"')");
       return false;
+    }
+
     
     // Convert the incoming acls that we know about to qualified forms, and reject the document
if
     // we don't know how to deal with its acls
@@ -554,8 +557,11 @@ public class HttpPoster
       // Reject documents that have security we don't know how to deal with in the Solr plugin!!
 Only safe thing to do.
       if (!aclType.equals(RepositoryDocument.SECURITY_TYPE_DOCUMENT) &&
         !aclType.equals(RepositoryDocument.SECURITY_TYPE_SHARE) &&
-        !aclType.startsWith(RepositoryDocument.SECURITY_TYPE_PARENT))
-        return false;
+        !aclType.startsWith(RepositoryDocument.SECURITY_TYPE_PARENT)){
+          activities.recordActivity(null,SolrConnector.INGEST_ACTIVITY,null,documentURI,activities.UNKNOWN_SECURITY,"Solr
connector rejected document that has security info which is unknown.");
+          return false;
+      }
+
     }
 
     try
@@ -940,9 +946,9 @@ public class HttpPoster
               (activityDetails.toLowerCase(Locale.ROOT).indexOf("broken pipe") != -1 ||
                 activityDetails.toLowerCase(Locale.ROOT).indexOf("connection reset") != -1
||
                 activityDetails.toLowerCase(Locale.ROOT).indexOf("target server failed to
respond") != -1))
-              activityCode = "SOLR REJECT";
+              activityCode = "SOLRREJECT";
             else
-              activityCode = "FAILED";
+              activityCode = e.getClass().getSimpleName().toUpperCase(Locale.ROOT);
 
             // Rethrow; will interpret at a higher level
             throw e;
@@ -966,7 +972,7 @@ public class HttpPoster
             return;
           
           activityStart = new Long(fullStartTime);
-          activityCode = "IO ERROR";
+          activityCode = ioe.getClass().getSimpleName().toUpperCase(Locale.ROOT);
           activityDetails = ioe.getMessage();
 
           // Log the error
@@ -1331,7 +1337,7 @@ public class HttpPoster
         catch (SolrServerException e)
         {
           activityStart = new Long(fullStartTime);
-          activityCode = "FAILED";
+          activityCode = e.getClass().getSimpleName().toUpperCase(Locale.ROOT);
           activityDetails = e.getMessage() +
             ((e.getCause() != null)?": "+e.getCause().getMessage():"");
 
@@ -1340,7 +1346,7 @@ public class HttpPoster
         catch (SolrException e)
         {
           activityStart = new Long(fullStartTime);
-          activityCode = "FAILED";
+          activityCode = e.getClass().getSimpleName().toUpperCase(Locale.ROOT);
           activityDetails = e.getMessage() +
             ((e.getCause() != null)?": "+e.getCause().getMessage():"");
 
@@ -1352,7 +1358,7 @@ public class HttpPoster
           Logging.ingest.warn("Error deleting document: "+ioe.getMessage(),ioe);
 
           activityStart = new Long(fullStartTime);
-          activityCode = "IO ERROR";
+          activityCode = ioe.getClass().getSimpleName().toUpperCase(Locale.ROOT);
           activityDetails = ioe.getMessage();
 
           throw ioe;



Mime
View raw message