manifoldcf-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kwri...@apache.org
Subject svn commit: r1632961 - in /manifoldcf/branches/dev_1x: ./ connectors/filesystem/connector/src/main/java/org/apache/manifoldcf/agents/output/filesystem/ connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/ framework/ framewo...
Date Sun, 19 Oct 2014 17:51:04 GMT
Author: kwright
Date: Sun Oct 19 17:51:04 2014
New Revision: 1632961

URL: http://svn.apache.org/r1632961
Log:
Pull up changes related to CONNECTORS-1077 from trunk

Modified:
    manifoldcf/branches/dev_1x/   (props changed)
    manifoldcf/branches/dev_1x/connectors/filesystem/connector/src/main/java/org/apache/manifoldcf/agents/output/filesystem/FileOutputConnector.java
    manifoldcf/branches/dev_1x/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputConnector.java
    manifoldcf/branches/dev_1x/framework/   (props changed)
    manifoldcf/branches/dev_1x/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/IOutputHistoryActivity.java

Propchange: manifoldcf/branches/dev_1x/
------------------------------------------------------------------------------
  Merged /manifoldcf/trunk:r1632844,1632847,1632854

Modified: manifoldcf/branches/dev_1x/connectors/filesystem/connector/src/main/java/org/apache/manifoldcf/agents/output/filesystem/FileOutputConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/connectors/filesystem/connector/src/main/java/org/apache/manifoldcf/agents/output/filesystem/FileOutputConnector.java?rev=1632961&r1=1632960&r2=1632961&view=diff
==============================================================================
--- manifoldcf/branches/dev_1x/connectors/filesystem/connector/src/main/java/org/apache/manifoldcf/agents/output/filesystem/FileOutputConnector.java
(original)
+++ manifoldcf/branches/dev_1x/connectors/filesystem/connector/src/main/java/org/apache/manifoldcf/agents/output/filesystem/FileOutputConnector.java
Sun Oct 19 17:51:04 2014
@@ -19,7 +19,6 @@
 package org.apache.manifoldcf.agents.output.filesystem;
 
 import java.io.File;
-import java.io.FileInputStream;
 import java.io.FileNotFoundException;
 import java.io.FileOutputStream;
 import java.io.IOException;
@@ -37,10 +36,7 @@ import java.util.Locale;
 import java.util.Map;
 import java.util.HashMap;
 
-import org.apache.manifoldcf.agents.interfaces.IOutputAddActivity;
-import org.apache.manifoldcf.agents.interfaces.IOutputRemoveActivity;
-import org.apache.manifoldcf.agents.interfaces.RepositoryDocument;
-import org.apache.manifoldcf.agents.interfaces.ServiceInterruption;
+import org.apache.manifoldcf.agents.interfaces.*;
 import org.apache.manifoldcf.agents.output.BaseOutputConnector;
 import org.apache.manifoldcf.agents.system.Logging;
 import org.apache.manifoldcf.core.interfaces.Specification;
@@ -48,7 +44,6 @@ import org.apache.manifoldcf.core.interf
 import org.apache.manifoldcf.core.interfaces.ConfigurationNode;
 import org.apache.manifoldcf.core.interfaces.IHTTPOutput;
 import org.apache.manifoldcf.core.interfaces.IPostParameters;
-import org.apache.manifoldcf.core.interfaces.IThreadContext;
 import org.apache.manifoldcf.core.interfaces.ManifoldCFException;
 import org.apache.manifoldcf.core.interfaces.SpecificationNode;
 import org.apache.manifoldcf.core.interfaces.VersionContext;
@@ -219,7 +214,10 @@ public class FileOutputConnector extends
             suffix++;
           }
           else
-            throw new ManifoldCFException("Could not create directory '"+newPath+"'.  Permission
issue?");
+          {
+              activities.recordActivity(null,INGEST_ACTIVITY,null,documentURI,activities.CREATED_DIRECTORY,"Couldn't
create directory ("+newPath+").");
+              throw new ManifoldCFException("Could not create directory '"+newPath+"'.  Permission
issue?");
+          }
         }
         // Directory successfully created!
         currentPath = newPath;
@@ -249,6 +247,7 @@ public class FileOutputConnector extends
             continue;
           }
           // Probably some other error
+          activities.recordActivity(null, INGEST_ACTIVITY, new Long(document.getBinaryLength()),
documentURI, activities.EXCEPTION, "Rejected due to FileNotFoundException.");
           throw new ManifoldCFException("Could not create file '"+outputPath+"': "+e.getMessage(),e);
         }
       }
@@ -259,8 +258,11 @@ public class FileOutputConnector extends
           */
         FileChannel channel = output.getChannel();
         FileLock lock = channel.tryLock();
-        if (lock == null)
+        if (lock == null){
+          activities.recordActivity(null,INGEST_ACTIVITY, new Long(document.getBinaryLength()),
documentURI,activities.EXCEPTION,"Rejected due to ServiceInterruptionException.");
           throw new ServiceInterruption("Could not lock file: '"+outputPath+"'",null,1000L,-1L,10,false);
+        }
+
 
         try {
 
@@ -290,15 +292,19 @@ public class FileOutputConnector extends
         }
       }
     } catch (URISyntaxException e) {
+      activities.recordActivity(null,INGEST_ACTIVITY,new Long(document.getBinaryLength()),
documentURI,activities.EXCEPTION,"Rejected due to URISyntaxException");
       handleURISyntaxException(e);
       return DOCUMENTSTATUS_REJECTED;
     } catch (FileNotFoundException e) {
+      activities.recordActivity(null,INGEST_ACTIVITY,new Long(document.getBinaryLength()),
documentURI,activities.EXCEPTION,"Rejected due to FileNotFoundException");
       handleFileNotFoundException(e);
       return DOCUMENTSTATUS_REJECTED;
     } catch (SecurityException e) {
+      activities.recordActivity(null,INGEST_ACTIVITY,new Long(document.getBinaryLength()),
documentURI,activities.EXCEPTION,"Rejected due to SecurityException");
       handleSecurityException(e);
       return DOCUMENTSTATUS_REJECTED;
     } catch (IOException e) {
+      activities.recordActivity(null,INGEST_ACTIVITY,new Long(document.getBinaryLength()),
documentURI ,activities.EXCEPTION,"Rejected due to IOException.");
       handleIOException(e);
       return DOCUMENTSTATUS_REJECTED;
     }
@@ -426,18 +432,23 @@ public class FileOutputConnector extends
         catch (FileNotFoundException e)
         {
           // Probably some other error
+          activities.recordActivity(null,REMOVE_ACTIVITY,null,documentURI,activities.EXCEPTION,"Couldn't
deleted due to FileNotFoundException");
           throw new ManifoldCFException("Could not zero out file '"+outputPath+"': "+e.getMessage(),e);
         }
       }
       // Just close it, to make a zero-length grave marker.
       output.close();
     } catch (URISyntaxException e) {
+      activities.recordActivity(null,REMOVE_ACTIVITY,null, documentURI,activities.EXCEPTION,"Rejected
due to URISyntaxException");
       handleURISyntaxException(e);
     } catch (FileNotFoundException e) {
+      activities.recordActivity(null,REMOVE_ACTIVITY,null, documentURI,activities.EXCEPTION,"Rejected
due to FileNotFoundException");
       handleFileNotFoundException(e);
     } catch (SecurityException e) {
+      activities.recordActivity(null,REMOVE_ACTIVITY,null, documentURI,activities.EXCEPTION,"Rejected
due to SecurityException");
       handleSecurityException(e);
     } catch (IOException e) {
+      activities.recordActivity(null,INGEST_ACTIVITY,null, documentURI ,activities.EXCEPTION,"Rejected
due to IOException.");
       handleIOException(e);
     }
 

Modified: manifoldcf/branches/dev_1x/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputConnector.java?rev=1632961&r1=1632960&r2=1632961&view=diff
==============================================================================
--- manifoldcf/branches/dev_1x/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputConnector.java
(original)
+++ manifoldcf/branches/dev_1x/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputConnector.java
Sun Oct 19 17:51:04 2014
@@ -286,10 +286,11 @@ public class HDFSOutputConnector extends
       Path path = new Path(strBuff.toString());
 
       Long startTime = new Long(System.currentTimeMillis());
-      createFile(path, document.getBinaryStream());
+      createFile(path, document.getBinaryStream(),activities,documentURI);
       activities.recordActivity(startTime, INGEST_ACTIVITY, new Long(document.getBinaryLength()),
documentURI, "OK", null);
       return DOCUMENTSTATUS_ACCEPTED;
     } catch (URISyntaxException e) {
+      activities.recordActivity(null,INGEST_ACTIVITY,new Long(document.getBinaryLength()),documentURI,activities.EXCEPTION,"Rejected
due to URISyntaxException");
       handleURISyntaxException(e);
       return DOCUMENTSTATUS_REJECTED;
     }
@@ -320,11 +321,13 @@ public class HDFSOutputConnector extends
       strBuff.append(documentURItoFilePath(documentURI));
       Path path = new Path(strBuff.toString());
       Long startTime = new Long(System.currentTimeMillis());
-      deleteFile(path);
+      deleteFile(path,activities,documentURI);
       activities.recordActivity(startTime, REMOVE_ACTIVITY, null, documentURI, "OK", null);
     } catch (JSONException e) {
+      activities.recordActivity(null,REMOVE_ACTIVITY,null,documentURI,activities.EXCEPTION,"Rejected
due to JSONException.");
       handleJSONException(e);
     } catch (URISyntaxException e) {
+      activities.recordActivity(null,REMOVE_ACTIVITY,null,documentURI,activities.EXCEPTION,"Rejected
due to URISyntaxException.");
       handleURISyntaxException(e);
     }
   }
@@ -669,7 +672,7 @@ public class HDFSOutputConnector extends
     }
   }
 
-  protected void createFile(Path path, InputStream input)
+  protected void createFile(Path path, InputStream input,IOutputAddActivity activities, String
documentURI)
     throws ManifoldCFException, ServiceInterruption {
     CreateFileThread t = new CreateFileThread(getSession(), path, input);
     try {
@@ -677,13 +680,17 @@ public class HDFSOutputConnector extends
       t.finishUp();
     } catch (InterruptedException e) {
       t.interrupt();
+      activities.recordActivity(null,INGEST_ACTIVITY,null,documentURI,activities.EXCEPTION,"Rejected
due to InterruptedException.");
       throw new ManifoldCFException("Interrupted: "+e.getMessage(),e,ManifoldCFException.INTERRUPTED);
     } catch (java.net.SocketTimeoutException e) {
+      activities.recordActivity(null,INGEST_ACTIVITY,null,documentURI,activities.EXCEPTION,"Rejected
due to SocketTimeoutException.");
       handleIOException(e);
     } catch (InterruptedIOException e) {
       t.interrupt();
+      activities.recordActivity(null,INGEST_ACTIVITY,null,documentURI,activities.EXCEPTION,"Rejected
due to InterruptedIOException.");
       handleIOException(e);
     } catch (IOException e) {
+      activities.recordActivity(null,INGEST_ACTIVITY,null,documentURI,activities.EXCEPTION,"Rejected
due to IOException.");
       handleIOException(e);
     }
   }
@@ -723,7 +730,7 @@ public class HDFSOutputConnector extends
     }
   }
 
-  protected void deleteFile(Path path)
+  protected void deleteFile(Path path,IOutputRemoveActivity activities,String documentURI)
     throws ManifoldCFException, ServiceInterruption {
     // Establish a session
     DeleteFileThread t = new DeleteFileThread(getSession(),path);
@@ -732,13 +739,17 @@ public class HDFSOutputConnector extends
       t.finishUp();
     } catch (InterruptedException e) {
       t.interrupt();
+      activities.recordActivity(null,REMOVE_ACTIVITY,null,documentURI,activities.EXCEPTION,"Rejected
due to InterruptedException.");
       throw new ManifoldCFException("Interrupted: "+e.getMessage(),e,ManifoldCFException.INTERRUPTED);
     } catch (java.net.SocketTimeoutException e) {
+      activities.recordActivity(null,REMOVE_ACTIVITY,null,documentURI,activities.EXCEPTION,"Rejected
due to SocketTimeoutException.");
       handleIOException(e);
     } catch (InterruptedIOException e) {
       t.interrupt();
+      activities.recordActivity(null,REMOVE_ACTIVITY,null,documentURI,activities.EXCEPTION,"Rejected
due to InterruptedIOException.");
       handleIOException(e);
     } catch (IOException e) {
+      activities.recordActivity(null,REMOVE_ACTIVITY,null,documentURI,activities.EXCEPTION,"Rejected
due to IOException.");
       handleIOException(e);
     }
   }

Propchange: manifoldcf/branches/dev_1x/framework/
------------------------------------------------------------------------------
  Merged /manifoldcf/trunk/framework:r1632844

Modified: manifoldcf/branches/dev_1x/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/IOutputHistoryActivity.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/IOutputHistoryActivity.java?rev=1632961&r1=1632960&r2=1632961&view=diff
==============================================================================
--- manifoldcf/branches/dev_1x/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/IOutputHistoryActivity.java
(original)
+++ manifoldcf/branches/dev_1x/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/IOutputHistoryActivity.java
Sun Oct 19 17:51:04 2014
@@ -27,6 +27,9 @@ public interface IOutputHistoryActivity
 {
   public static final String _rcsid = "@(#)$Id: IOutputHistoryActivity.java 988245 2010-08-23
18:39:35Z kwright $";
 
+  public static final String EXCEPTION = "EXCEPTION";
+  public static final String CREATED_DIRECTORY = "CREATEDDIRECTORY";
+
   /** Record time-stamped information about the activity of the output connector.
   *@param startTime is either null or the time since the start of epoch in milliseconds (Jan
1, 1970).  Every
   *       activity has an associated time; the startTime field records when the activity
began.  A null value



Mime
View raw message