manifoldcf-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kwri...@apache.org
Subject svn commit: r1611760 - in /manifoldcf/branches/CONNECTORS-989/framework: agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/ agents/src/main/java/org/apache/manifoldcf/agents/interfaces/ pull-agent/src/main/java/org/apache/manifoldcf/c...
Date Fri, 18 Jul 2014 18:37:49 GMT
Author: kwright
Date: Fri Jul 18 18:37:49 2014
New Revision: 1611760

URL: http://svn.apache.org/r1611760
Log:
Begin process of getting component IDs into connector, where decisions will be made

Modified:
    manifoldcf/branches/CONNECTORS-989/framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/IncrementalIngester.java
    manifoldcf/branches/CONNECTORS-989/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/IngestStatuses.java
    manifoldcf/branches/CONNECTORS-989/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StufferThread.java

Modified: manifoldcf/branches/CONNECTORS-989/framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/IncrementalIngester.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-989/framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/IncrementalIngester.java?rev=1611760&r1=1611759&r2=1611760&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-989/framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/IncrementalIngester.java
(original)
+++ manifoldcf/branches/CONNECTORS-989/framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/IncrementalIngester.java
Fri Jul 18 18:37:49 2014
@@ -1511,7 +1511,8 @@ public class IncrementalIngester extends
         if (authorityName == null)
           authorityName = "";
         int indexValue = position.intValue();
-        rval.addStatus(identifierClasses[indexValue],identifierHashes[indexValue],outputConnectionName,
+        // MHL
+        rval.addStatus(identifierClasses[indexValue],identifierHashes[indexValue],null,outputConnectionName,
           new DocumentIngestStatus(lastVersion,lastTransformationVersion,lastOutputVersion,paramVersion,authorityName));
       }
     }

Modified: manifoldcf/branches/CONNECTORS-989/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/IngestStatuses.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-989/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/IngestStatuses.java?rev=1611760&r1=1611759&r2=1611760&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-989/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/IngestStatuses.java
(original)
+++ manifoldcf/branches/CONNECTORS-989/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/IngestStatuses.java
Fri Jul 18 18:37:49 2014
@@ -28,7 +28,7 @@ public class IngestStatuses
 {
   public static final String _rcsid = "@(#)$Id$";
 
-  protected final Map<OutputKey,DocumentIngestStatus> statuses = new HashMap<OutputKey,DocumentIngestStatus>();
+  protected final Map<OutputKey,Map<String,DocumentIngestStatus>> statuses =
new HashMap<OutputKey,Map<String,DocumentIngestStatus>>();
   
   public IngestStatuses()
   {
@@ -37,23 +37,50 @@ public class IngestStatuses
   /** Add a status record.
   *@param documentClass is the document class.
   *@param documentIDHash is the document id's hash value.
+  *@param componentIDHash is the component id hash value, if any.
   *@param outputConnectionName is the output connection name.
   *@param status is the status record.
   */
-  public void addStatus(String documentClass, String documentIDHash, String outputConnectionName,
DocumentIngestStatus status)
+  public void addStatus(String documentClass, String documentIDHash, String componentIDHash,
String outputConnectionName, DocumentIngestStatus status)
   {
-    statuses.put(new OutputKey(documentClass,documentIDHash,outputConnectionName),status);
+    if (componentIDHash == null)
+      componentIDHash = "";
+    OutputKey ok = new OutputKey(documentClass,documentIDHash,outputConnectionName);
+    Map<String,DocumentIngestStatus> map = statuses.get(ok);
+    if (map == null)
+    {
+      map = new HashMap<String,DocumentIngestStatus>();
+      statuses.put(ok,map);
+    }
+    map.put(componentIDHash,status);
+  }
+  
+  /** Get the set of component hashes for a given output.
+  *@param documentClass is the document class.
+  *@param documentIDHash is the document id's hash value.
+  *@param outputConnectionName is the output connection name.
+  *@return the set of component hashes (empty string meaning no component).
+  */
+  public Set<String> componentIterator(String documentClass, String documentIDHash,
String outputConnectionName)
+  {
+    Map<String,DocumentIngestStatus> map = statuses.get(new OutputKey(documentClass,documentIDHash,outputConnectionName));
+    if (map == null)
+      return new HashSet<String>();
+    return map.keySet();
   }
   
   /** Retrieve a status record.
   *@param documentClass is the document class.
   *@param documentIDHash is the document id's hash value.
+  *@param componentIDHash is the component id hash value, if any.
   *@param outputConnectionName is the output connection name.
   *@return the status record, if record.
   */
-  public DocumentIngestStatus getStatus(String documentClass, String documentIDHash, String
outputConnectionName)
+  public DocumentIngestStatus getStatus(String documentClass, String documentIDHash, String
componentIDHash, String outputConnectionName)
   {
-    return statuses.get(new OutputKey(documentClass,documentIDHash,outputConnectionName));
+    if (componentIDHash == null)
+      componentIDHash = "";
+    return statuses.get(new OutputKey(documentClass,documentIDHash,outputConnectionName)).get(componentIDHash);
   }
 
   protected static class OutputKey

Modified: manifoldcf/branches/CONNECTORS-989/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StufferThread.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-989/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StufferThread.java?rev=1611760&r1=1611759&r2=1611760&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-989/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StufferThread.java
(original)
+++ manifoldcf/branches/CONNECTORS-989/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StufferThread.java
Fri Jul 18 18:37:49 2014
@@ -257,7 +257,8 @@ public class StufferThread extends Threa
             for (int j = 0; j < pipelineSpecifications[i].getOutputCount(); j++)
             {
               String outputName = pipelineSpecifications[i].getStageConnectionName(pipelineSpecifications[i].getOutputStage(j));
-              DocumentIngestStatus status = statuses.getStatus(documentClasses[i],documentIDHashes[i],outputName);
+              // MHL
+              DocumentIngestStatus status = statuses.getStatus(documentClasses[i],documentIDHashes[i],null,outputName);
               if (status != null)
                 versions[i].put(outputName,status);
             }



Mime
View raw message