manifoldcf-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kwri...@apache.org
Subject svn commit: r1599193 - in /manifoldcf/branches/CONNECTORS-946/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler: interfaces/IJobDescription.java jobs/JobDescription.java jobs/PipelineManager.java
Date Mon, 02 Jun 2014 12:43:36 GMT
Author: kwright
Date: Mon Jun  2 12:43:35 2014
New Revision: 1599193

URL: http://svn.apache.org/r1599193
Log:
Revise pipeline stage portion of JobDescription to make more sense

Modified:
    manifoldcf/branches/CONNECTORS-946/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/IJobDescription.java
    manifoldcf/branches/CONNECTORS-946/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobDescription.java
    manifoldcf/branches/CONNECTORS-946/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/PipelineManager.java

Modified: manifoldcf/branches/CONNECTORS-946/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/IJobDescription.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-946/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/IJobDescription.java?rev=1599193&r1=1599192&r2=1599193&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-946/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/IJobDescription.java
(original)
+++ manifoldcf/branches/CONNECTORS-946/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/IJobDescription.java
Mon Jun  2 12:43:35 2014
@@ -96,26 +96,40 @@ public interface IJobDescription
   public void clearPipeline();
   
   /** Add a pipeline connection.
-  *@param pipelineConnectionName is the name of the pipeline connection to add.
+  *@param pipelineStageConnectionName is the name of the pipeline connection to add.
+  *@param pipelineStageDescription is a description of the pipeline stage being added.
+  *@return the empty output specification for this pipeline stage.
   */
-  public void addPipelineConnection(String pipelineConnectionName);
+  public OutputSpecification addPipelineStage(String pipelineStageConnectionName, String
pipelineStageDescription);
   
   /** Get a count of pipeline connections.
   *@return the current number of pipeline connections.
   */
-  public int countPipelineConnections();
+  public int countPipelineStages();
   
   /** Get a specific pipeline connection name.
-  *@param index is the index of the pipeline whose connection name to get.
+  *@param index is the index of the pipeline stage whose connection name to get.
   *@return the name of the connection.
   */
-  public String getPipelineConnectionName(int index);
-  
-  /** Get a specific pipeline connection specification.
-  *@param index is the index of the pipeline whose specification is needed.
+  public String getPipelineStageConnectionName(int index);
+
+  /** Get a specific pipeline stage description.
+  *@param index is the index of the pipeline stage whose description to get.
+  *@return the name of the connection.
+  */
+  public String getPipelineStageDescription(int index);
+
+  /** Get a specific pipeline stage specification.
+  *@param index is the index of the pipeline stage whose specification is needed.
   *@return the specification for the connection.
   */
-  public OutputSpecification getPipelineSpecification(int index);
+  public OutputSpecification getPipelineStageSpecification(int index);
+
+  /** Delete a pipeline stage.
+  *@param index is the index of the pipeline stage to delete.
+  */
+  public void deletePipelineStage(int index);
+  
 
   /** Set the job type.
   *@param type is the type (as an integer).

Modified: manifoldcf/branches/CONNECTORS-946/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobDescription.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-946/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobDescription.java?rev=1599193&r1=1599192&r2=1599193&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-946/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobDescription.java
(original)
+++ manifoldcf/branches/CONNECTORS-946/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobDescription.java
Mon Jun  2 12:43:35 2014
@@ -44,7 +44,7 @@ public class JobDescription implements I
   protected String description = null;
   protected String outputConnectionName = null;
   protected String connectionName = null;
-  protected final List<String> pipelineConnectionNames = new ArrayList<String>();
+  protected final List<PipelineStage> pipelineStages = new ArrayList<PipelineStage>();
   protected int type = TYPE_CONTINUOUS;
   protected int startMethod = START_WINDOWBEGIN;
   protected int priority = 5;
@@ -73,9 +73,6 @@ public class JobDescription implements I
   // Document specification
   protected DocumentSpecification documentSpecification = new DocumentSpecification();
 
-  // Transformation specifications
-  protected final List<OutputSpecification> pipelineSpecifications = new ArrayList<OutputSpecification>();
-  
   // Hop count filters.
   protected HashMap hopCountFilters = new HashMap();
 
@@ -100,10 +97,14 @@ public class JobDescription implements I
     rval.id = id;
     rval.isNew = isNew;
     rval.outputConnectionName = outputConnectionName;
+    // Direct modification of this object is possible - so it also has to know if it is read-only!!
+    rval.outputSpecification = outputSpecification.duplicate(readOnly);
     rval.connectionName = connectionName;
-    for (String pipelineConnectionName : pipelineConnectionNames)
+    // Direct modification of this object is possible - so it also has to know if it is read-only!!
+    rval.documentSpecification = documentSpecification.duplicate(readOnly);
+    for (PipelineStage pipelineStage : pipelineStages)
     {
-      rval.pipelineConnectionNames.add(pipelineConnectionName);
+      rval.pipelineStages.add(new PipelineStage(pipelineStage.getConnectionName(),pipelineStage.getDescription(),pipelineStage.getSpecification().duplicate(readOnly)));
     }
     rval.description = description;
     rval.type = type;
@@ -132,15 +133,6 @@ public class JobDescription implements I
         rval.addForcedMetadataValue(forcedParamName,value);
       }
     }
-    // Direct modification of this object is possible - so it also has to know if it is read-only!!
-    rval.outputSpecification = outputSpecification.duplicate(readOnly);
-    // Direct modification of this object is possible - so it also has to know if it is read-only!!
-    rval.documentSpecification = documentSpecification.duplicate(readOnly);
-    // Duplicate the pipeline specifications
-    for (OutputSpecification pipelineSpecification : pipelineSpecifications)
-    {
-      rval.pipelineSpecifications.add(pipelineSpecification.duplicate(readOnly));
-    }
     rval.readOnly = readOnly;
     return rval;
   }
@@ -261,41 +253,70 @@ public class JobDescription implements I
   {
     if (readOnly)
       throw new IllegalStateException("Attempt to change read-only object");
-    pipelineConnectionNames.clear();
-    pipelineSpecifications.clear();
+    pipelineStages.clear();
   }
   
-  /** Add a pipeline connection */
+  /** Add a pipeline connection.
+  *@param pipelineStageConnectionName is the name of the pipeline connection to add.
+  *@param pipelineStageDescription is a description of the pipeline stage being added.
+  *@return the empty output specification for this pipeline stage.
+  */
   @Override
-  public void addPipelineConnection(String pipelineConnectionName)
+  public OutputSpecification addPipelineStage(String pipelineStageConnectionName, String
pipelineStageDescription)
   {
     if (readOnly)
       throw new IllegalStateException("Attempt to change read-only object");
-    pipelineConnectionNames.add(pipelineConnectionName);
-    pipelineSpecifications.add(new OutputSpecification());
+    PipelineStage ps = new PipelineStage(pipelineStageConnectionName,pipelineStageDescription);
+    pipelineStages.add(ps);
+    return ps.getSpecification();
   }
   
-  /** Get a count of pipeline connections */
+  /** Get a count of pipeline stages */
   @Override
-  public int countPipelineConnections()
+  public int countPipelineStages()
   {
-    return pipelineConnectionNames.size();
+    return pipelineStages.size();
   }
   
-  /** Get a specific pipeline connection name */
+  /** Get a specific pipeline connection name.
+  *@param index is the index of the pipeline stage whose connection name to get.
+  *@return the name of the connection.
+  */
   @Override
-  public String getPipelineConnectionName(int index)
+  public String getPipelineStageConnectionName(int index)
   {
-    return pipelineConnectionNames.get(index);
+    return pipelineStages.get(index).getConnectionName();
   }
   
-  /** Get a specific pipeline connection specification */
+  /** Get a specific pipeline stage description.
+  *@param index is the index of the pipeline stage whose description to get.
+  *@return the name of the connection.
+  */
   @Override
-  public OutputSpecification getPipelineSpecification(int index)
+  public String getPipelineStageDescription(int index)
   {
-    return pipelineSpecifications.get(index);
+    return pipelineStages.get(index).getDescription();
   }
-  
+
+  /** Get a specific pipeline stage specification.
+  *@param index is the index of the pipeline stage whose specification is needed.
+  *@return the specification for the connection.
+  */
+  @Override
+  public OutputSpecification getPipelineStageSpecification(int index)
+  {
+    return pipelineStages.get(index).getSpecification();
+  }
+
+  /** Delete a pipeline stage.
+  *@param index is the index of the pipeline stage to delete.
+  */
+  @Override
+  public void deletePipelineStage(int index)
+  {
+    pipelineStages.remove(index);
+  }
+
   /** Set the job type.
   *@param type is the type (as an integer).
   */
@@ -598,4 +619,40 @@ public class JobDescription implements I
     rval.add(value);
   }
 
+  protected static class PipelineStage
+  {
+    protected final String connectionName;
+    protected final String description;
+    protected final OutputSpecification specification;
+    
+    public PipelineStage(String connectionName, String description)
+    {
+      this.connectionName = connectionName;
+      this.description = description;
+      this.specification = new OutputSpecification();
+    }
+
+    public PipelineStage(String connectionName, String description, OutputSpecification spec)
+    {
+      this.connectionName = connectionName;
+      this.description = description;
+      this.specification = spec;
+    }
+    
+    public OutputSpecification getSpecification()
+    {
+      return specification;
+    }
+    
+    public String getConnectionName()
+    {
+      return connectionName;
+    }
+    
+    public String getDescription()
+    {
+      return description;
+    }
+  }
+  
 }

Modified: manifoldcf/branches/CONNECTORS-946/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/PipelineManager.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-946/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/PipelineManager.java?rev=1599193&r1=1599192&r2=1599193&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-946/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/PipelineManager.java
(original)
+++ manifoldcf/branches/CONNECTORS-946/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/PipelineManager.java
Mon Jun  2 12:43:35 2014
@@ -34,6 +34,7 @@ import java.util.*;
 * <tr><td>ownerid</td><td>BIGINT</td><td>Reference:jobs.id</td></tr>
 * <tr><td>ordinal</td><td>BIGINT</td><td></td></tr>
 * <tr><td>transformationname</td><td>VARCHAR(32)</td><td></td></tr>
+* <tr><td>transformationdesc</td><td>VARCHAR(255)</td><td></td></tr>
 * <tr><td>transformationspec</td><td>LONGTEXT</td><td></td></tr>
 * </table>
 * <br><br>
@@ -47,6 +48,7 @@ public class PipelineManager extends org
   public final static String ownerIDField = "ownerid";
   public final static String ordinalField = "ordinal";
   public final static String transformationNameField = "transformationname";
+  public final static String transformationDescriptionField = "transformationdesc";
   public final static String transformationSpecField = "transformationspec";
 
   /** Constructor.
@@ -76,6 +78,7 @@ public class PipelineManager extends org
         map.put(ownerIDField,new ColumnDescription("BIGINT",false,false,ownerTable,owningTablePrimaryKey,false));
         map.put(ordinalField,new ColumnDescription("BIGINT",false,false,null,null,false));
         map.put(transformationNameField,new ColumnDescription("VARCHAR(32)",false,false,transformationTableName,transformationNameField,false));
+        map.put(transformationDescriptionField,new ColumnDescription("VARCHAR(255)",false,true,null,null,false));
         map.put(transformationSpecField,new ColumnDescription("LONGTEXT",false,true,null,null,false));
         performCreate(map,null);
       }
@@ -132,12 +135,11 @@ public class PipelineManager extends org
     {
       IResultRow row = set.getRow(i);
       Long ownerID = (Long)row.getValue(ownerIDField);
-      Long ordinal = (Long)row.getValue(ordinalField);
       String transformationName = (String)row.getValue(transformationNameField);
+      String transformationDesc = (String)row.getValue(transformationDescriptionField);
       String transformationSpec = (String)row.getValue(transformationSpecField);
       JobDescription jd = returnValues.get(ownerID);
-      jd.addPipelineConnection(transformationName);
-      jd.getPipelineSpecification((int)ordinal.longValue()).fromXML(transformationSpec);
+      jd.addPipelineStage(transformationName,transformationDesc).fromXML(transformationSpec);
     }
   }
 
@@ -152,14 +154,17 @@ public class PipelineManager extends org
     try
     {
       HashMap map = new HashMap();
-      for (int i = 0; i < job.countPipelineConnections(); i++)
+      for (int i = 0; i < job.countPipelineStages(); i++)
       {
-        String pipelineConnectionName = job.getPipelineConnectionName(i);
-        OutputSpecification os = job.getPipelineSpecification(i);
+        String pipelineConnectionName = job.getPipelineStageConnectionName(i);
+        String pipelineStageDescription = job.getPipelineStageDescription(i);
+        OutputSpecification os = job.getPipelineStageSpecification(i);
         map.clear();
         map.put(ownerIDField,ownerID);
         map.put(ordinalField,new Long((long)i));
         map.put(transformationNameField,pipelineConnectionName);
+        if (pipelineStageDescription != null && pipelineStageDescription.length()
> 0)
+          map.put(transformationDescriptionField,pipelineStageDescription);
         map.put(transformationSpecField,os.toXML());
         performInsert(map,null);
         i++;



Mime
View raw message