manifoldcf-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kwri...@apache.org
Subject svn commit: r1600552 - in /manifoldcf/branches/CONNECTORS-946/framework/agents/src/main/java/org/apache/manifoldcf/agents: incrementalingest/IncrementalIngester.java interfaces/IIncrementalIngester.java
Date Thu, 05 Jun 2014 00:46:29 GMT
Author: kwright
Date: Thu Jun  5 00:46:29 2014
New Revision: 1600552

URL: http://svn.apache.org/r1600552
Log:
Add some modified IIncrementalIngester methods

Modified:
    manifoldcf/branches/CONNECTORS-946/framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/IncrementalIngester.java
    manifoldcf/branches/CONNECTORS-946/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/IIncrementalIngester.java

Modified: manifoldcf/branches/CONNECTORS-946/framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/IncrementalIngester.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-946/framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/IncrementalIngester.java?rev=1600552&r1=1600551&r2=1600552&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-946/framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/IncrementalIngester.java
(original)
+++ manifoldcf/branches/CONNECTORS-946/framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/IncrementalIngester.java
Thu Jun  5 00:46:29 2014
@@ -74,13 +74,17 @@ public class IncrementalIngester extends
   protected final static String authorityNameField = "authorityname";
 
   // Thread context.
-  protected IThreadContext threadContext;
+  protected final IThreadContext threadContext;
   // Lock manager.
-  protected ILockManager lockManager;
+  protected final ILockManager lockManager;
   // Output connection manager
-  protected IOutputConnectionManager connectionManager;
+  protected final IOutputConnectionManager connectionManager;
   // Output connector pool manager
-  protected IOutputConnectorPool outputConnectorPool;
+  protected final IOutputConnectorPool outputConnectorPool;
+  // Transformation connection manager
+  protected final ITransformationConnectionManager transformationConnectionManager;
+  // Transformation connector pool manager
+  protected final ITransformationConnectorPool transformationConnectorPool;
   
   /** Constructor.
   */
@@ -92,6 +96,8 @@ public class IncrementalIngester extends
     lockManager = LockManagerFactory.make(threadContext);
     connectionManager = OutputConnectionManagerFactory.make(threadContext);
     outputConnectorPool = OutputConnectorPoolFactory.make(threadContext);
+    transformationConnectionManager = TransformationConnectionManagerFactory.make(threadContext);
+    transformationConnectorPool = TransformationConnectorPoolFactory.make(threadContext);
   }
 
   /** Install the incremental ingestion manager.
@@ -207,6 +213,27 @@ public class IncrementalIngester extends
   public boolean checkMimeTypeIndexable(String outputConnectionName, String outputDescription,
String mimeType)
     throws ManifoldCFException, ServiceInterruption
   {
+    return checkMimeTypeIndexable(new String[0], new String[0],
+      outputConnectionName, outputDescription,
+      mimeType);
+  }
+
+  /** Check if a mime type is indexable.
+  *@param transformationConnectionNames is the ordered list of transformation connection
names.
+  *@param transformationDescriptions is the ordered list of transformation description strings.
+  *@param outputConnectionName is the name of the output connection associated with this
action.
+  *@param outputDescription is the output description string.
+  *@param mimeType is the mime type to check.
+  *@return true if the mimeType is indexable.
+  */
+  @Override
+  public boolean checkMimeTypeIndexable(
+    String[] transformationConnectionNames, String[] transformationDescriptions,
+    String outputConnectionName, String outputDescription,
+    String mimeType)
+    throws ManifoldCFException, ServiceInterruption
+  {
+    // MHL to handle the pipeline
     IOutputConnection connection = connectionManager.load(outputConnectionName);
     IOutputConnector connector = outputConnectorPool.grab(connection);
     if (connector == null)
@@ -232,6 +259,27 @@ public class IncrementalIngester extends
   public boolean checkDocumentIndexable(String outputConnectionName, String outputDescription,
File localFile)
     throws ManifoldCFException, ServiceInterruption
   {
+    return checkDocumentIndexable(new String[0], new String[0],
+      outputConnectionName, outputDescription,
+      localFile);
+  }
+  
+  /** Check if a file is indexable.
+  *@param transformationConnectionNames is the ordered list of transformation connection
names.
+  *@param transformationDescriptions is the ordered list of transformation description strings.
+  *@param outputConnectionName is the name of the output connection associated with this
action.
+  *@param outputDescription is the output description string.
+  *@param localFile is the local file to check.
+  *@return true if the local file is indexable.
+  */
+  @Override
+  public boolean checkDocumentIndexable(
+    String[] transformationConnectionNames, String[] transformationDescriptions,
+    String outputConnectionName, String outputDescription,
+    File localFile)
+    throws ManifoldCFException, ServiceInterruption
+  {
+    // MHL
     IOutputConnection connection = connectionManager.load(outputConnectionName);
     IOutputConnector connector = outputConnectorPool.grab(connection);
     if (connector == null)
@@ -258,6 +306,28 @@ public class IncrementalIngester extends
   public boolean checkLengthIndexable(String outputConnectionName, String outputDescription,
long length)
     throws ManifoldCFException, ServiceInterruption
   {
+    return checkLengthIndexable(new String[0], new String[0],
+      outputConnectionName, outputDescription,
+      length);
+  }
+  
+  /** Pre-determine whether a document's length is indexable by this connector.  This method
is used by participating repository connectors
+  * to help filter out documents that are too long to be indexable.
+  *@param transformationConnectionNames is the ordered list of transformation connection
names.
+  *@param transformationDescriptions is the ordered list of transformation description strings.
+  *@param outputConnectionName is the name of the output connection associated with this
action.
+  *@param outputDescription is the output description string.
+  *@param length is the length of the document.
+  *@return true if the file is indexable.
+  */
+  @Override
+  public boolean checkLengthIndexable(
+    String[] transformationConnectionNames, String[] transformationDescriptions,
+    String outputConnectionName, String outputDescription,
+    long length)
+    throws ManifoldCFException, ServiceInterruption
+  {
+    // MHL
     IOutputConnection connection = connectionManager.load(outputConnectionName);
     IOutputConnector connector = outputConnectorPool.grab(connection);
     if (connector == null)
@@ -284,6 +354,28 @@ public class IncrementalIngester extends
   public boolean checkURLIndexable(String outputConnectionName, String outputDescription,
String url)
     throws ManifoldCFException, ServiceInterruption
   {
+    return checkURLIndexable(new String[0], new String[0],
+      outputConnectionName, outputDescription,
+      url);
+  }
+  
+  /** Pre-determine whether a document's URL is indexable by this connector.  This method
is used by participating repository connectors
+  * to help filter out documents that not indexable.
+  *@param transformationConnectionNames is the ordered list of transformation connection
names.
+  *@param transformationDescriptions is the ordered list of transformation description strings.
+  *@param outputConnectionName is the name of the output connection associated with this
action.
+  *@param outputDescription is the output description string.
+  *@param url is the url of the document.
+  *@return true if the file is indexable.
+  */
+  @Override
+  public boolean checkURLIndexable(
+    String[] transformationConnectionNames, String[] transformationDescriptions,
+    String outputConnectionName, String outputDescription,
+    String url)
+    throws ManifoldCFException, ServiceInterruption
+  {
+    // MHL
     IOutputConnection connection = connectionManager.load(outputConnectionName);
     IOutputConnector connector = outputConnectorPool.grab(connection);
     if (connector == null)
@@ -324,6 +416,30 @@ public class IncrementalIngester extends
 
   }
 
+  /** Get a transformation version string for a document.
+  *@param transformationConnectionName is the name of the transformation connection associated
with this action.
+  *@param spec is the transformation specification.
+  *@return the description string.
+  */
+  @Override
+  public String getTransformationDescription(String transformationConnectionName, OutputSpecification
spec)
+    throws ManifoldCFException, ServiceInterruption
+  {
+    ITransformationConnection connection = transformationConnectionManager.load(transformationConnectionName);
+    ITransformationConnector connector = transformationConnectorPool.grab(connection);
+    if (connector == null)
+      // The connector is not installed; treat this as a service interruption.
+      throw new ServiceInterruption("Transformation connector not installed",0L);
+    try
+    {
+      return connector.getPipelineDescription(spec);
+    }
+    finally
+    {
+      transformationConnectorPool.release(connection,connector);
+    }
+  }
+
   /** Record a document version, but don't ingest it.
   * The purpose of this method is to keep track of the frequency at which ingestion "attempts"
take place.
   * ServiceInterruption is thrown if this action must be rescheduled.

Modified: manifoldcf/branches/CONNECTORS-946/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/IIncrementalIngester.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-946/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/IIncrementalIngester.java?rev=1600552&r1=1600551&r2=1600552&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-946/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/IIncrementalIngester.java
(original)
+++ manifoldcf/branches/CONNECTORS-946/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/IIncrementalIngester.java
Thu Jun  5 00:46:29 2014
@@ -68,6 +68,28 @@ public interface IIncrementalIngester
   public String getOutputDescription(String outputConnectionName, OutputSpecification spec)
     throws ManifoldCFException, ServiceInterruption;
 
+  /** Get a transformation version string for a document.
+  *@param transformationConnectionName is the name of the transformation connection associated
with this action.
+  *@param spec is the transformation specification.
+  *@return the description string.
+  */
+  public String getTransformationDescription(String transformationConnectionName, OutputSpecification
spec)
+    throws ManifoldCFException, ServiceInterruption;
+
+  /** Check if a mime type is indexable.
+  *@param transformationConnectionNames is the ordered list of transformation connection
names.
+  *@param transformationDescriptions is the ordered list of transformation description strings.
+  *@param outputConnectionName is the name of the output connection associated with this
action.
+  *@param outputDescription is the output description string.
+  *@param mimeType is the mime type to check.
+  *@return true if the mimeType is indexable.
+  */
+  public boolean checkMimeTypeIndexable(
+    String[] transformationConnectionNames, String[] transformationDescriptions,
+    String outputConnectionName, String outputDescription,
+    String mimeType)
+    throws ManifoldCFException, ServiceInterruption;
+
   /** Check if a mime type is indexable.
   *@param outputConnectionName is the name of the output connection associated with this
action.
   *@param outputDescription is the output description string.
@@ -78,6 +100,20 @@ public interface IIncrementalIngester
     throws ManifoldCFException, ServiceInterruption;
 
   /** Check if a file is indexable.
+  *@param transformationConnectionNames is the ordered list of transformation connection
names.
+  *@param transformationDescriptions is the ordered list of transformation description strings.
+  *@param outputConnectionName is the name of the output connection associated with this
action.
+  *@param outputDescription is the output description string.
+  *@param localFile is the local file to check.
+  *@return true if the local file is indexable.
+  */
+  public boolean checkDocumentIndexable(
+    String[] transformationConnectionNames, String[] transformationDescriptions,
+    String outputConnectionName, String outputDescription,
+    File localFile)
+    throws ManifoldCFException, ServiceInterruption;
+
+  /** Check if a file is indexable.
   *@param outputConnectionName is the name of the output connection associated with this
action.
   *@param outputDescription is the output description string.
   *@param localFile is the local file to check.
@@ -88,6 +124,21 @@ public interface IIncrementalIngester
 
   /** Pre-determine whether a document's length is indexable by this connector.  This method
is used by participating repository connectors
   * to help filter out documents that are too long to be indexable.
+  *@param transformationConnectionNames is the ordered list of transformation connection
names.
+  *@param transformationDescriptions is the ordered list of transformation description strings.
+  *@param outputConnectionName is the name of the output connection associated with this
action.
+  *@param outputDescription is the output description string.
+  *@param length is the length of the document.
+  *@return true if the file is indexable.
+  */
+  public boolean checkLengthIndexable(
+    String[] transformationConnectionNames, String[] transformationDescriptions,
+    String outputConnectionName, String outputDescription,
+    long length)
+    throws ManifoldCFException, ServiceInterruption;
+
+  /** Pre-determine whether a document's length is indexable by this connector.  This method
is used by participating repository connectors
+  * to help filter out documents that are too long to be indexable.
   *@param outputConnectionName is the name of the output connection associated with this
action.
   *@param outputDescription is the output description string.
   *@param length is the length of the document.
@@ -98,6 +149,21 @@ public interface IIncrementalIngester
 
   /** Pre-determine whether a document's URL is indexable by this connector.  This method
is used by participating repository connectors
   * to help filter out documents that not indexable.
+  *@param transformationConnectionNames is the ordered list of transformation connection
names.
+  *@param transformationDescriptions is the ordered list of transformation description strings.
+  *@param outputConnectionName is the name of the output connection associated with this
action.
+  *@param outputDescription is the output description string.
+  *@param url is the url of the document.
+  *@return true if the file is indexable.
+  */
+  public boolean checkURLIndexable(
+    String[] transformationConnectionNames, String[] transformationDescriptions,
+    String outputConnectionName, String outputDescription,
+    String url)
+    throws ManifoldCFException, ServiceInterruption;
+
+  /** Pre-determine whether a document's URL is indexable by this connector.  This method
is used by participating repository connectors
+  * to help filter out documents that not indexable.
   *@param outputConnectionName is the name of the output connection associated with this
action.
   *@param outputDescription is the output description string.
   *@param url is the url of the document.



Mime
View raw message