manifoldcf-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kwri...@apache.org
Subject svn commit: r1602056 - in /manifoldcf/branches/CONNECTORS-962/framework: agents/src/main/java/org/apache/manifoldcf/agents/interfaces/IPipelineSpecification.java pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/WorkerThread.java
Date Thu, 12 Jun 2014 01:27:52 GMT
Author: kwright
Date: Thu Jun 12 01:27:51 2014
New Revision: 1602056

URL: http://svn.apache.org/r1602056
Log:
Define a class which can be used to specify a pipeline to the incremental ingester.

Added:
    manifoldcf/branches/CONNECTORS-962/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/IPipelineSpecification.java
  (with props)
Modified:
    manifoldcf/branches/CONNECTORS-962/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/WorkerThread.java

Added: manifoldcf/branches/CONNECTORS-962/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/IPipelineSpecification.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-962/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/IPipelineSpecification.java?rev=1602056&view=auto
==============================================================================
--- manifoldcf/branches/CONNECTORS-962/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/IPipelineSpecification.java
(added)
+++ manifoldcf/branches/CONNECTORS-962/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/IPipelineSpecification.java
Thu Jun 12 01:27:51 2014
@@ -0,0 +1,48 @@
+/* $Id$ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.agents.interfaces;
+
+import org.apache.manifoldcf.core.interfaces.*;
+
+/** This interface describes a multi-output pipeline.  Each stage of the pipeline is
+* given a rank number, and dependencies between stages refer to that rank number.
+*/
+public interface IPipelineSpecification
+{
+  public static final String _rcsid = "@(#)$Id$";
+
+  /** Find children of a given pipeline stage.  Pass -1 to find the children of the root
stage.
+  *@param stage is the stage index to get the children of.
+  *@return the pipeline stages that represent those children.
+  */
+  public int[] getStageChildren(int stage);
+  
+  /** Get the connection name for a pipeline stage.
+  *@param stage is the stage to get the connection name for.
+  *@return the connection name for that stage.
+  */
+  public String getStageConnectionName(int stage);
+  
+  /** Check if a stage is an output stage.
+  *@param stage is the stage to check.
+  *@return true if the stage represents an output connection.
+  */
+  public boolean checkStageOutputConnection(int stage);
+  
+}

Propchange: manifoldcf/branches/CONNECTORS-962/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/IPipelineSpecification.java
------------------------------------------------------------------------------
    svn:eol-style = native

Propchange: manifoldcf/branches/CONNECTORS-962/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/IPipelineSpecification.java
------------------------------------------------------------------------------
    svn:keywords = Id

Modified: manifoldcf/branches/CONNECTORS-962/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/WorkerThread.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-962/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/WorkerThread.java?rev=1602056&r1=1602055&r2=1602056&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-962/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/WorkerThread.java
(original)
+++ manifoldcf/branches/CONNECTORS-962/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/WorkerThread.java
Thu Jun 12 01:27:51 2014
@@ -2785,4 +2785,54 @@ public class WorkerThread extends Thread
       true);
   }
 
+  /** Pipeline specification implementation.
+  */
+  protected static class PipelineSpecification implements IPipelineSpecification
+  {
+    protected final String[] transformationConnectionNames;
+    protected final String outputConnectionName;
+    
+    public PipelineSpecification(IJobDescription job)
+    {
+      transformationConnectionNames = new String[job.countPipelineStages()];
+      outputConnectionName = job.getOutputConnectionName();
+      for (int i = 0; i < transformationConnectionNames.length; i++)
+      {
+        transformationConnectionNames[i] = job.getPipelineStageConnectionName(i);
+      }
+    }
+    
+    /** Find children of a given pipeline stage.  Pass -1 to find the children of the root
stage.
+    *@param stage is the stage index to get the children of.
+    *@return the pipeline stages that represent those children.
+    */
+    public int[] getStageChildren(int stage)
+    {
+      if (stage < transformationConnectionNames.length + 1)
+        return new int[]{stage + 1};
+      return new int[0];
+    }
+    
+    /** Get the connection name for a pipeline stage.
+    *@param stage is the stage to get the connection name for.
+    *@return the connection name for that stage.
+    */
+    public String getStageConnectionName(int stage)
+    {
+      if (stage < transformationConnectionNames.length)
+        return transformationConnectionNames[stage];
+      return outputConnectionName;
+    }
+    
+    /** Check if a stage is an output stage.
+    *@param stage is the stage to check.
+    *@return true if the stage represents an output connection.
+    */
+    public boolean checkStageOutputConnection(int stage)
+    {
+      return stage == transformationConnectionNames.length;
+    }
+
+  }
+  
 }



Mime
View raw message