manifoldcf-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kwri...@apache.org
Subject svn commit: r1607853 - in /manifoldcf/branches/CONNECTORS-989/framework: agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/ agents/src/main/java/org/apache/manifoldcf/agents/interfaces/ pull-agent/src/main/java/org/apache/manifoldcf/c...
Date Fri, 04 Jul 2014 13:34:44 GMT
Author: kwright
Date: Fri Jul  4 13:34:44 2014
New Revision: 1607853

URL: http://svn.apache.org/r1607853
Log:
Initial work on various APIs

Modified:
    manifoldcf/branches/CONNECTORS-989/framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/IncrementalIngester.java
    manifoldcf/branches/CONNECTORS-989/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/IIncrementalIngester.java
    manifoldcf/branches/CONNECTORS-989/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/OutputKey.java
    manifoldcf/branches/CONNECTORS-989/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/PipelineSpecificationWithVersions.java
    manifoldcf/branches/CONNECTORS-989/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/QueuedDocument.java
    manifoldcf/branches/CONNECTORS-989/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StufferThread.java
    manifoldcf/branches/CONNECTORS-989/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/WorkerThread.java

Modified: manifoldcf/branches/CONNECTORS-989/framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/IncrementalIngester.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-989/framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/IncrementalIngester.java?rev=1607853&r1=1607852&r2=1607853&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-989/framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/IncrementalIngester.java
(original)
+++ manifoldcf/branches/CONNECTORS-989/framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/IncrementalIngester.java
Fri Jul  4 13:34:44 2014
@@ -687,6 +687,40 @@ public class IncrementalIngester extends
     sb.append(delim);
   }
 
+  /** Note the start of processing of a set of documents.  This method does whatever is needed
to handle the
+  * bookkeeping for the documents' indexing records, and the indexing records for multiple
virtual child documents.
+  * The documents must all have the same identifier class.
+  *@param pipelineSpecificationBasic is the basic pipeline specification for the set of documents.
+  *@param identifierClass is the name of the space in which the identifier hash should be
interpreted.
+  *@param identifierHashes are the document identifier hashes that are about to be processed.
+  */
+  @Override
+  public void beginDocumentProcessing(
+    IPipelineSpecificationBasic pipelineSpecificationBasic,
+    String identifierClass, String[] identifierHashes)
+    throws ManifoldCFException
+  {
+    // MHL
+  }
+
+  /** Note the end of processing of a set of documents.  This method completes bookkeeping
for the documents'
+  * indexing records, and the indexing records for multiple virtual child documents.  This
method may cause
+  * documents to be removed from the specified output connections, should that be indicated.
+  *@param pipelineSpecificationBasic is the basic pipeline specification for the set of documents.
+  *@param identifierClass is the name of the space in which the identifier hash should be
interpreted.
+  *@param identifierHashes are the document identifier hashes that are about to be processed.
+  *@param activities is the object to use to log the details of any removals.  May be null.
+  */
+  @Override
+  public void endDocumentProcessing(
+    IPipelineSpecificationBasic pipelineSpecificationBasic,
+    String identifierClass, String[] identifierHashes,
+    IOutputRemoveActivity activities)
+    throws ManifoldCFException, ServiceInterruption
+  {
+    // MHL
+  }
+
   /** Record a document version, but don't ingest it.
   * The purpose of this method is to keep track of the frequency at which ingestion "attempts"
take place.
   * ServiceInterruption is thrown if this action must be rescheduled.
@@ -707,7 +741,7 @@ public class IncrementalIngester extends
   {
     documentRecord(
       new RuntPipelineSpecificationBasic(outputConnectionName),
-      identifierClass, identifierHash,
+      identifierClass, identifierHash, null,
       documentVersion,
       recordTime, activities);
   }
@@ -718,6 +752,7 @@ public class IncrementalIngester extends
   *@param pipelineSpecificationBasic is the basic pipeline specification needed.
   *@param identifierClass is the name of the space in which the identifier hash should be
interpreted.
   *@param identifierHash is the hashed document identifier.
+  *@param childIdentifierHash is the hashed virtual child document identifier.  Pass null
if this is a primary record.
   *@param documentVersion is the document version.
   *@param recordTime is the time at which the recording took place, in milliseconds since
epoch.
   *@param activities is the object used in case a document needs to be removed from the output
index as the result of this operation.
@@ -725,11 +760,12 @@ public class IncrementalIngester extends
   @Override
   public void documentRecord(
     IPipelineSpecificationBasic pipelineSpecificationBasic,
-    String identifierClass, String identifierHash,
+    String identifierClass, String identifierHash, String childIdentifierHash,
     String documentVersion, long recordTime,
     IOutputActivity activities)
     throws ManifoldCFException, ServiceInterruption
   {
+    // MHL
     String docKey = makeKey(identifierClass,identifierHash);
 
     String[] outputConnectionNames = extractOutputConnectionNames(pipelineSpecificationBasic);
@@ -917,7 +953,7 @@ public class IncrementalIngester extends
       return documentIngest(
         new RuntPipelineSpecificationWithVersions(outputConnectionName,outputVersion,
           "","","","",""),
-        identifierClass, identifierHash,
+        identifierClass, identifierHash, null,
         documentVersion,
         parameterVersion,
         authorityName,
@@ -1026,6 +1062,7 @@ public class IncrementalIngester extends
   *@param pipelineSpecificationWithVersions is the pipeline specification with already-fetched
output versioning information.
   *@param identifierClass is the name of the space in which the identifier hash should be
interpreted.
   *@param identifierHash is the hashed document identifier.
+  *@param childIdentifierHash is the hashed virtual child document identifier.  Pass null
if this is a primary record.
   *@param documentVersion is the document version.
   *@param parameterVersion is the version string for the forced parameters.
   *@param authorityName is the name of the authority associated with the document, if any.
@@ -1039,7 +1076,7 @@ public class IncrementalIngester extends
   @Override
   public boolean documentIngest(
     IPipelineSpecificationWithVersions pipelineSpecificationWithVersions,
-    String identifierClass, String identifierHash,
+    String identifierClass, String identifierHash, String childIdentifierHash,
     String documentVersion,
     String parameterVersion,
     String authorityName,
@@ -1048,6 +1085,8 @@ public class IncrementalIngester extends
     IOutputActivity activities)
     throws ManifoldCFException, ServiceInterruption, IOException
   {
+    // MHL
+
     PipelineConnectionsWithVersions pipelineConnectionsWithVersions = new PipelineConnectionsWithVersions(pipelineSpecificationWithVersions);
     
     String docKey = makeKey(identifierClass,identifierHash);
@@ -1894,7 +1933,8 @@ public class IncrementalIngester extends
         if (authorityName == null)
           authorityName = "";
         int indexValue = position.intValue();
-        rval.put(new OutputKey(identifierClasses[indexValue],identifierHashes[indexValue],outputConnectionName),
+        // MHL
+        rval.put(new OutputKey(identifierClasses[indexValue],identifierHashes[indexValue],null,outputConnectionName),
           new DocumentIngestStatus(lastVersion,lastTransformationVersion,lastOutputVersion,paramVersion,authorityName));
       }
     }

Modified: manifoldcf/branches/CONNECTORS-989/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/IIncrementalIngester.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-989/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/IIncrementalIngester.java?rev=1607853&r1=1607852&r2=1607853&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-989/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/IIncrementalIngester.java
(original)
+++ manifoldcf/branches/CONNECTORS-989/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/IIncrementalIngester.java
Fri Jul  4 13:34:44 2014
@@ -200,6 +200,32 @@ public interface IIncrementalIngester
     String newParameterVersion,
     String newAuthorityNameString);
 
+  /** Note the start of processing of a set of documents.  This method does whatever is needed
to handle the
+  * bookkeeping for the documents' indexing records, and the indexing records for multiple
virtual child documents.
+  * The documents must all have the same identifier class.
+  *@param pipelineSpecificationBasic is the basic pipeline specification for the set of documents.
+  *@param identifierClass is the name of the space in which the identifier hash should be
interpreted.
+  *@param identifierHashes are the document identifier hashes that are about to be processed.
+  */
+  public void beginDocumentProcessing(
+    IPipelineSpecificationBasic pipelineSpecificationBasic,
+    String identifierClass, String[] identifierHashes)
+    throws ManifoldCFException;
+
+  /** Note the end of processing of a set of documents.  This method completes bookkeeping
for the documents'
+  * indexing records, and the indexing records for multiple virtual child documents.  This
method may cause
+  * documents to be removed from the specified output connections, should that be indicated.
+  *@param pipelineSpecificationBasic is the basic pipeline specification for the set of documents.
+  *@param identifierClass is the name of the space in which the identifier hash should be
interpreted.
+  *@param identifierHashes are the document identifier hashes that are about to be processed.
+  *@param activities is the object to use to log the details of any removals.  May be null.
+  */
+  public void endDocumentProcessing(
+    IPipelineSpecificationBasic pipelineSpecificationBasic,
+    String identifierClass, String[] identifierHashes,
+    IOutputRemoveActivity activities)
+    throws ManifoldCFException, ServiceInterruption;
+
   /** Record a document version, but don't ingest it.
   * The purpose of this method is to keep track of the frequency at which ingestion "attempts"
take place.
   * ServiceInterruption is thrown if this action must be rescheduled.
@@ -223,13 +249,14 @@ public interface IIncrementalIngester
   *@param pipelineSpecificationBasic is the basic pipeline specification needed.
   *@param identifierClass is the name of the space in which the identifier hash should be
interpreted.
   *@param identifierHash is the hashed document identifier.
+  *@param childIdentifierHash is the hashed virtual child document identifier.  Pass null
if this is a primary record.
   *@param documentVersion is the document version.
   *@param recordTime is the time at which the recording took place, in milliseconds since
epoch.
   *@param activities is the object used in case a document needs to be removed from the output
index as the result of this operation.
   */
   public void documentRecord(
     IPipelineSpecificationBasic pipelineSpecificationBasic,
-    String identifierClass, String identifierHash,
+    String identifierClass, String identifierHash, String childIdentifierHash,
     String documentVersion, long recordTime,
     IOutputActivity activities)
     throws ManifoldCFException, ServiceInterruption;
@@ -300,6 +327,7 @@ public interface IIncrementalIngester
   *@param pipelineSpecificationWithVersions is the pipeline specification with already-fetched
output versioning information.
   *@param identifierClass is the name of the space in which the identifier hash should be
interpreted.
   *@param identifierHash is the hashed document identifier.
+  *@param childIdentifierHash is the hashed virtual child document identifier.  Pass null
if this is a primary record.
   *@param documentVersion is the document version.
   *@param parameterVersion is the version string for the forced parameters.
   *@param authorityName is the name of the authority associated with the document, if any.
@@ -312,7 +340,7 @@ public interface IIncrementalIngester
   */
   public boolean documentIngest(
     IPipelineSpecificationWithVersions pipelineSpecificationWithVersions,
-    String identifierClass, String identifierHash,
+    String identifierClass, String identifierHash, String childIdentifierHash,
     String documentVersion,
     String parameterVersion,
     String authorityName,

Modified: manifoldcf/branches/CONNECTORS-989/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/OutputKey.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-989/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/OutputKey.java?rev=1607853&r1=1607852&r2=1607853&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-989/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/OutputKey.java
(original)
+++ manifoldcf/branches/CONNECTORS-989/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/OutputKey.java
Fri Jul  4 13:34:44 2014
@@ -32,14 +32,16 @@ public class OutputKey
 
   protected final String documentClass;
   protected final String documentIDHash;
+  protected final String childIDHash;
   protected final String outputConnectionName;
   
   /** Constructor */
-  public OutputKey(String documentClass, String documentIDHash, String outputConnectionName)
+  public OutputKey(String documentClass, String documentIDHash, String childIDHash, String
outputConnectionName)
   {
     // Identifying information
     this.documentClass = documentClass;
     this.documentIDHash = documentIDHash;
+    this.childIDHash = childIDHash;
     this.outputConnectionName = outputConnectionName;
   }
 
@@ -55,6 +57,12 @@ public class OutputKey
     return documentIDHash;
   }
   
+  /** Get the child ID hash */
+  public String getChildIDHash()
+  {
+    return childIDHash;
+  }
+  
   /** Get the output connection name */
   public String getOutputConnectionName()
   {
@@ -63,7 +71,9 @@ public class OutputKey
   
   public int hashCode()
   {
-    return documentClass.hashCode() + documentIDHash.hashCode() + outputConnectionName.hashCode();
+    return documentClass.hashCode() + documentIDHash.hashCode() + 
+      ((childIDHash==null)?0:childIDHash.hashCode()) +
+      outputConnectionName.hashCode();
   }
   
   public boolean equals(Object o)
@@ -73,6 +83,7 @@ public class OutputKey
     OutputKey dis = (OutputKey)o;
     return dis.documentClass.equals(documentClass) &&
       dis.documentIDHash.equals(documentIDHash) &&
+      (((dis.childIDHash == null || childIDHash == null) && dis.childIDHash == childIDHash)
|| (dis.childIDHash != null && childIDHash != null && dis.childIDHash.equals(childIDHash)))
&&
       dis.outputConnectionName.equals(outputConnectionName);
   }
       

Modified: manifoldcf/branches/CONNECTORS-989/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/PipelineSpecificationWithVersions.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-989/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/PipelineSpecificationWithVersions.java?rev=1607853&r1=1607852&r2=1607853&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-989/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/PipelineSpecificationWithVersions.java
(original)
+++ manifoldcf/branches/CONNECTORS-989/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/PipelineSpecificationWithVersions.java
Fri Jul  4 13:34:44 2014
@@ -49,7 +49,8 @@ public class PipelineSpecificationWithVe
   protected DocumentIngestStatus getStatus(int index)
   {
     IPipelineSpecificationBasic basic = pipelineSpecification.getBasicPipelineSpecification();
-    return queuedDocument.getLastIngestedStatus(basic.getStageConnectionName(basic.getOutputStage(index)));
+    // MHL
+    return queuedDocument.getLastIngestedStatus(basic.getStageConnectionName(basic.getOutputStage(index)),"");
   }
   
   /** For a given output index, return a document version string.

Modified: manifoldcf/branches/CONNECTORS-989/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/QueuedDocument.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-989/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/QueuedDocument.java?rev=1607853&r1=1607852&r2=1607853&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-989/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/QueuedDocument.java
(original)
+++ manifoldcf/branches/CONNECTORS-989/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/QueuedDocument.java
Fri Jul  4 13:34:44 2014
@@ -38,8 +38,8 @@ public class QueuedDocument
 
   /** The document description. */
   protected final DocumentDescription documentDescription;
-  /** The last ingested status, null meaning "never ingested". */
-  protected final Map<String,DocumentIngestStatus> lastIngestedStatus;
+  /** The last ingested status for all child records, arranged by output connection name.
null means "never ingested". */
+  protected final Map<String,Map<String,DocumentIngestStatus>> lastIngestedStatus;
   /** The binnames for the document, according to the connector */
   protected final String[] binNames;
   /** This flag indicates whether the document has been processed or not. */
@@ -47,10 +47,11 @@ public class QueuedDocument
 
   /** Constructor.
   *@param documentDescription is the document description.
-  *@param lastIngestedStatus is the document's last ingested status.
+  *@param lastIngestedStatus is the document's last ingested status.  Map is keyed by output
connection name, and child map is
+  *   keyed by child document id hash value, or a blank value for a primary row.
   *@param binNames are the bins associated with the document.
   */
-  public QueuedDocument(DocumentDescription documentDescription, Map<String,DocumentIngestStatus>
lastIngestedStatus, String[] binNames)
+  public QueuedDocument(DocumentDescription documentDescription, Map<String,Map<String,DocumentIngestStatus>>
lastIngestedStatus, String[] binNames)
   {
     this.documentDescription = documentDescription;
     this.lastIngestedStatus = lastIngestedStatus;
@@ -65,15 +66,34 @@ public class QueuedDocument
     return documentDescription;
   }
 
+  /** Get an iterator over the child document IDs available for this document identifier
and a specified
+  * output connection name.
+  *@param outputConnectionName is the name of the output connection.
+  *@return an iterator over child document IDs.  For the primary ID, an empty value will
be returned by this iterator.
+  */
+  public Iterator<String> getLastIngestedChildIDs(String outputConnectionName)
+  {
+    if (lastIngestedStatus == null)
+      return new HashSet<String>().iterator();
+    Map<String,DocumentIngestStatus> rval = lastIngestedStatus.get(outputConnectionName);
+    if (rval == null)
+      return new HashSet<String>().iterator();
+    return rval.keySet().iterator();
+  }
+  
   /** Get the last ingested status.
   *@param outputConnectionName is the name of the output connection.
+  *@param childIDHash is the child identifier hash, using an empty value for the primary.
   *@return the last ingested status for that output, or null if not found.
   */
-  public DocumentIngestStatus getLastIngestedStatus(String outputConnectionName)
+  public DocumentIngestStatus getLastIngestedStatus(String outputConnectionName, String childIDHash)
   {
     if (lastIngestedStatus == null)
       return null;
-    return lastIngestedStatus.get(outputConnectionName);
+    Map<String,DocumentIngestStatus> rval = lastIngestedStatus.get(outputConnectionName);
+    if (rval == null)
+      return null;
+    return rval.get(childIDHash);
   }
 
   /** Return true if there are *any* last ingested records.

Modified: manifoldcf/branches/CONNECTORS-989/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StufferThread.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-989/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StufferThread.java?rev=1607853&r1=1607852&r2=1607853&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-989/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StufferThread.java
(original)
+++ manifoldcf/branches/CONNECTORS-989/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StufferThread.java
Fri Jul  4 13:34:44 2014
@@ -250,18 +250,36 @@ public class StufferThread extends Threa
 
           Map<OutputKey,DocumentIngestStatus> statuses = new HashMap<OutputKey,DocumentIngestStatus>();
           ingester.getPipelineDocumentIngestDataMultiple(statuses,pipelineSpecifications,documentClasses,documentIDHashes);
-          // Break apart the result.
+          
+          // Process the results.  Ideally, we'd just use a get operation to find the right
document status, but the child id messes
+          // that up, so we need to create a way of finding the right map to put things into.
+          Map<String,Integer> indexMap = new HashMap<String,Integer>();
           for (int i = 0; i < descs.length; i++)
           {
-            versions[i] = new HashMap<String,DocumentIngestStatus>();
-            for (int j = 0; j < pipelineSpecifications[i].getOutputCount(); j++)
-            {
-              String outputName = pipelineSpecifications[i].getStageConnectionName(pipelineSpecifications[i].getOutputStage(j));
-              OutputKey key = new OutputKey(documentClasses[i],documentIDHashes[i],outputName);
-              DocumentIngestStatus status = statuses.get(key);
-              if (status != null)
-                versions[i].put(outputName,status);
-            }
+            indexMap.put(documentClasses[i] + documentIDHashes[i],new Integer(i));
+            versions[i] = null;
+          }
+          
+          // Populate each map in the way we need it.
+          for (OutputKey key : statuses.keySet())
+          {
+            int index = indexMap.get(key.getDocumentClass() + key.getDocumentIDHash()).intValue();
+            Map<String,Map<String,DocumentIngestStatus>> entry = versions[index];
+            if (entry == null)
+            {
+              entry = new HashMap<String,Map<String,DocumentIngestStatus>>();
+              versions[index] = entry;
+            }
+            Map<String,DocumentIngestStatus> childAndStatus = entry.get(key.getOutputConnectionName());
+            if (childAndStatus == null)
+            {
+              childAndStatus = new HashMap<String,DocumentIngestStatus>();
+              entry.put(key.getOutputConnectionName(),childAndStatus);
+            }
+            String childIDHash = key.getChildIDHash();
+            if (childIDHash == null)
+              childIDHash = "";
+            childAndStatus.put(childIDHash,statuses.get(key));
           }
 
           // We need to go through the list, and segregate them by job, so the individual
@@ -335,7 +353,7 @@ public class StufferThread extends Threa
               binNames = new String[]{""};
             }
 
-            QueuedDocument qd = new QueuedDocument(descs[i],(Map<String,DocumentIngestStatus>)versions[i],binNames);
+            QueuedDocument qd = new QueuedDocument(descs[i],(Map<String,Map<String,DocumentIngestStatus>>)versions[i],binNames);
 
             // Grab the arraylist that's there, or create it.
             List<QueuedDocument> set = documentSets.get(jobID);

Modified: manifoldcf/branches/CONNECTORS-989/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/WorkerThread.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-989/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/WorkerThread.java?rev=1607853&r1=1607852&r2=1607853&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-989/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/WorkerThread.java
(original)
+++ manifoldcf/branches/CONNECTORS-989/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/WorkerThread.java
Fri Jul  4 13:34:44 2014
@@ -307,7 +307,8 @@ public class WorkerThread extends Thread
                       QueuedDocument qd = activeDocuments.get(i);
                       currentDocIDHashArray[i] = qd.getDocumentDescription().getDocumentIdentifierHash();
                       currentDocIDArray[i] = qd.getDocumentDescription().getDocumentIdentifier();
-                      DocumentIngestStatus dis = qd.getLastIngestedStatus(lastIndexedOutputConnectionName);
+                      // MHL
+                      DocumentIngestStatus dis = qd.getLastIngestedStatus(lastIndexedOutputConnectionName,"");
                       if (dis == null)
                         oldVersionStringArray[i] = null;
                       else
@@ -996,6 +997,7 @@ public class WorkerThread extends Thread
     {
       QueuedDocument qd = deleteList.get(i);
       // See if we need to delete from index
+      // MHL
       if (qd.anyLastIngestedRecords())
       {
         // Queue up to issue deletion
@@ -1746,9 +1748,10 @@ public class WorkerThread extends Thread
       throws ManifoldCFException, ServiceInterruption
     {
       String documentIdentifierHash = ManifoldCF.hash(documentIdentifier);
+      // MHL
       ingester.documentRecord(
         pipelineSpecification.getBasicPipelineSpecification(),
-        connectionName,documentIdentifierHash,
+        connectionName,documentIdentifierHash,null,
         version,currentTime,ingestLogger);
     }
 
@@ -1816,9 +1819,10 @@ public class WorkerThread extends Thread
       }
         
       // First, we need to add into the metadata the stuff from the job description.
+      // MHL
       ingester.documentIngest(
         fetchPipelineSpecifications.get(documentIdentifierHash),
-        connectionName,documentIdentifierHash,
+        connectionName,documentIdentifierHash,null,
         version,parameterVersion,
         connection.getACLAuthority(),
         data,currentTime,



Mime
View raw message