manifoldcf-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kwri...@apache.org
Subject svn commit: r1640944 - in /manifoldcf/branches/dev_1x: ./ CHANGES.txt framework/ framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/IncrementalIngester.java
Date Fri, 21 Nov 2014 15:42:36 GMT
Author: kwright
Date: Fri Nov 21 15:42:36 2014
New Revision: 1640944

URL: http://svn.apache.org/r1640944
Log:
Pull up fix for CONNECTORS-1111 from trunk.

Modified:
    manifoldcf/branches/dev_1x/   (props changed)
    manifoldcf/branches/dev_1x/CHANGES.txt
    manifoldcf/branches/dev_1x/framework/   (props changed)
    manifoldcf/branches/dev_1x/framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/IncrementalIngester.java

Propchange: manifoldcf/branches/dev_1x/
------------------------------------------------------------------------------
  Merged /manifoldcf/trunk:r1640941-1640942

Modified: manifoldcf/branches/dev_1x/CHANGES.txt
URL: http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/CHANGES.txt?rev=1640944&r1=1640943&r2=1640944&view=diff
==============================================================================
--- manifoldcf/branches/dev_1x/CHANGES.txt (original)
+++ manifoldcf/branches/dev_1x/CHANGES.txt Fri Nov 21 15:42:36 2014
@@ -3,6 +3,9 @@ $Id$
 
 ======================= 1.8-dev =====================
 
+CONNECTORS-1111: Component cleanup did not work properly.
+(Markus Schuch, Karl Wright)
+
 CONNECTORS-1110: Component documents getting deleted wrongly
 when additional components added.
 (Markus Schuch, Karl Wright)

Propchange: manifoldcf/branches/dev_1x/framework/
------------------------------------------------------------------------------
  Merged /manifoldcf/trunk/framework:r1640941

Modified: manifoldcf/branches/dev_1x/framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/IncrementalIngester.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/IncrementalIngester.java?rev=1640944&r1=1640943&r2=1640944&view=diff
==============================================================================
--- manifoldcf/branches/dev_1x/framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/IncrementalIngester.java
(original)
+++ manifoldcf/branches/dev_1x/framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/IncrementalIngester.java
Fri Nov 21 15:42:36 2014
@@ -1052,26 +1052,26 @@ public class IncrementalIngester extends
       // No transactions.  Time for the operation may exceed transaction timeout.
 
       // Obtain the current URIs of all of these.
-      DeleteInfo[] uris = getDocumentURIMultiple(outputConnectionName,identifierClasses,identifierHashes);
+      List<DeleteInfo> uris = getDocumentURIMultiple(outputConnectionName,identifierClasses,identifierHashes);
 
       // Grab critical section locks so that we can't attempt to ingest at the same time
we are deleting.
       // (This guarantees that when this operation is complete the database reflects reality.)
       int validURIcount = 0;
-      for (int i = 0; i < uris.length; i++)
+      for (DeleteInfo uri : uris)
       {
-        if (uris[i] != null && uris[i].getURI() != null)
+        if (uri.getURI() != null)
           validURIcount++;
       }
       String[] lockArray = new String[validURIcount];
       String[] validURIArray = new String[validURIcount];
       String[] validURIHashArray = new String[validURIcount];
       validURIcount = 0;
-      for (int i = 0; i < uris.length; i++)
+      for (DeleteInfo uri : uris)
       {
-        if (uris[i] != null && uris[i].getURI() != null)
+        if (uri.getURI() != null)
         {
-          validURIArray[validURIcount] = uris[i].getURI();
-          validURIHashArray[validURIcount] = uris[i].getURIHash();
+          validURIArray[validURIcount] = uri.getURI();
+          validURIHashArray[validURIcount] = uri.getURIHash();
           lockArray[validURIcount] = outputConnectionName+":"+validURIHashArray[validURIcount];
           validURIcount++;
         }
@@ -1081,10 +1081,10 @@ public class IncrementalIngester extends
       try
       {
         // Fetch the document URIs for the listed documents
-        for (int i = 0; i < uris.length; i++)
+        for (DeleteInfo uri : uris)
         {
-          if (uris[i] != null && uris[i].getURI() != null)
-            removeDocument(connection,uris[i].getURI(),uris[i].getOutputVersion(),activities);
+          if (uri.getURI() != null)
+            removeDocument(connection,uri.getURI(),uri.getOutputVersion(),activities);
         }
 
         // Now, get rid of all rows that match the given uris.
@@ -1268,26 +1268,26 @@ public class IncrementalIngester extends
       // No transactions.  Time for the operation may exceed transaction timeout.
 
       // Obtain the current URIs of all of these.
-      DeleteInfo[] uris = getDocumentURIMultiple(outputConnectionName,identifierClasses,identifierHashes,componentHash);
+      List<DeleteInfo> uris = getDocumentURIMultiple(outputConnectionName,identifierClasses,identifierHashes,componentHash);
 
       // Grab critical section locks so that we can't attempt to ingest at the same time
we are deleting.
       // (This guarantees that when this operation is complete the database reflects reality.)
       int validURIcount = 0;
-      for (int i = 0; i < uris.length; i++)
+      for (DeleteInfo uri : uris)
       {
-        if (uris[i] != null && uris[i].getURI() != null)
+        if (uri.getURI() != null)
           validURIcount++;
       }
       String[] lockArray = new String[validURIcount];
       String[] validURIArray = new String[validURIcount];
       String[] validURIHashArray = new String[validURIcount];
       validURIcount = 0;
-      for (int i = 0; i < uris.length; i++)
+      for (DeleteInfo uri : uris)
       {
-        if (uris[i] != null && uris[i].getURI() != null)
+        if (uri.getURI() != null)
         {
-          validURIArray[validURIcount] = uris[i].getURI();
-          validURIHashArray[validURIcount] = uris[i].getURIHash();
+          validURIArray[validURIcount] = uri.getURI();
+          validURIHashArray[validURIcount] = uri.getURIHash();
           lockArray[validURIcount] = outputConnectionName+":"+validURIHashArray[validURIcount];
           validURIcount++;
         }
@@ -1297,10 +1297,10 @@ public class IncrementalIngester extends
       try
       {
         // Fetch the document URIs for the listed documents
-        for (int i = 0; i < uris.length; i++)
+        for (DeleteInfo uri : uris)
         {
-          if (uris[i] != null && uris[i].getURI() != null)
-            removeDocument(connection,uris[i].getURI(),uris[i].getOutputVersion(),activities);
+          if (uri.getURI() != null)
+            removeDocument(connection,uri.getURI(),uri.getOutputVersion(),activities);
         }
 
         // Now, get rid of all rows that match the given uris.
@@ -1615,37 +1615,29 @@ public class IncrementalIngester extends
   *@return the array of current document uri's.  Null returned for identifiers
   * that don't exist in the index.
   */
-  protected DeleteInfo[] getDocumentURIMultiple(String outputConnectionName, String[] identifierClasses,
String[] identifierHashes)
+  protected List<DeleteInfo> getDocumentURIMultiple(String outputConnectionName, String[]
identifierClasses, String[] identifierHashes)
     throws ManifoldCFException
   {
-    DeleteInfo[] rval = new DeleteInfo[identifierHashes.length];
-    Map<String,Integer> map = new HashMap<String,Integer>();
-    for (int i = 0; i < identifierHashes.length; i++)
-    {
-      map.put(makeKey(identifierClasses[i],identifierHashes[i]),new Integer(i));
-      rval[i] = null;
-    }
-
+    List<DeleteInfo> rval = new ArrayList<DeleteInfo>();
     beginTransaction();
     try
     {
       List<String> list = new ArrayList<String>();
       int maxCount = maxClauseDocumentURIChunk(outputConnectionName);
       int j = 0;
-      Iterator<String> iter = map.keySet().iterator();
-      while (iter.hasNext())
+      for (int i = 0; i < identifierHashes.length; i++)
       {
         if (j == maxCount)
         {
-          getDocumentURIChunk(rval,map,outputConnectionName,list);
+          getDocumentURIChunk(rval,outputConnectionName,list);
           j = 0;
           list.clear();
         }
-        list.add(iter.next());
+        list.add(makeKey(identifierClasses[i],identifierHashes[i]));
         j++;
       }
       if (j > 0)
-        getDocumentURIChunk(rval,map,outputConnectionName,list);
+        getDocumentURIChunk(rval,outputConnectionName,list);
       return rval;
     }
     catch (ManifoldCFException e)
@@ -1672,37 +1664,29 @@ public class IncrementalIngester extends
   *@return the array of current document uri's.  Null returned for identifiers
   * that don't exist in the index.
   */
-  protected DeleteInfo[] getDocumentURIMultiple(String outputConnectionName, String[] identifierClasses,
String[] identifierHashes, String componentHash)
+  protected List<DeleteInfo> getDocumentURIMultiple(String outputConnectionName, String[]
identifierClasses, String[] identifierHashes, String componentHash)
     throws ManifoldCFException
   {
-    DeleteInfo[] rval = new DeleteInfo[identifierHashes.length];
-    Map<String,Integer> map = new HashMap<String,Integer>();
-    for (int i = 0; i < identifierHashes.length; i++)
-    {
-      map.put(makeKey(identifierClasses[i],identifierHashes[i]),new Integer(i));
-      rval[i] = null;
-    }
-
+    List<DeleteInfo> rval = new ArrayList<DeleteInfo>();
     beginTransaction();
     try
     {
       List<String> list = new ArrayList<String>();
       int maxCount = maxClauseDocumentURIChunk(outputConnectionName,componentHash);
       int j = 0;
-      Iterator<String> iter = map.keySet().iterator();
-      while (iter.hasNext())
+      for (int i = 0; i < identifierHashes.length; i++)
       {
         if (j == maxCount)
         {
-          getDocumentURIChunk(rval,map,outputConnectionName,list,componentHash);
+          getDocumentURIChunk(rval,outputConnectionName,list,componentHash);
           j = 0;
           list.clear();
         }
-        list.add(iter.next());
+        list.add(makeKey(identifierClasses[i],identifierHashes[i]));
         j++;
       }
       if (j > 0)
-        getDocumentURIChunk(rval,map,outputConnectionName,list,componentHash);
+        getDocumentURIChunk(rval,outputConnectionName,list,componentHash);
       return rval;
     }
     catch (ManifoldCFException e)
@@ -2264,7 +2248,7 @@ public class IncrementalIngester extends
   *@param clause is the in clause for the query.
   *@param list are the doc keys for the query.
   */
-  protected void getDocumentURIChunk(DeleteInfo[] rval, Map<String,Integer> map, String
outputConnectionName,
+  protected void getDocumentURIChunk(List<DeleteInfo> rval, String outputConnectionName,
     List<String> list)
     throws ManifoldCFException
   {
@@ -2280,19 +2264,15 @@ public class IncrementalIngester extends
     for (int i = 0; i < set.getRowCount(); i++)
     {
       IResultRow row = set.getRow(i);
-      String docHash = row.getValue(docKeyField).toString();
-      Integer position = map.get(docHash);
-      if (position != null)
-      {
-        String lastURI = (String)row.getValue(docURIField);
-        if (lastURI != null && lastURI.length() == 0)
-          lastURI = null;
-        String lastURIHash = (String)row.getValue(uriHashField);
-        if (lastURIHash != null && lastURIHash.length() == 0)
-          lastURIHash = null;
-        String lastOutputVersion = (String)row.getValue(lastOutputVersionField);
-        rval[position.intValue()] = new DeleteInfo(lastURI,lastURIHash,lastOutputVersion);
-      }
+      //String docHash = row.getValue(docKeyField).toString();
+      String lastURI = (String)row.getValue(docURIField);
+      if (lastURI != null && lastURI.length() == 0)
+        lastURI = null;
+      String lastURIHash = (String)row.getValue(uriHashField);
+      if (lastURIHash != null && lastURIHash.length() == 0)
+        lastURIHash = null;
+      String lastOutputVersion = (String)row.getValue(lastOutputVersionField);
+      rval.add(new DeleteInfo(lastURI,lastURIHash,lastOutputVersion));
     }
   }
 
@@ -2312,7 +2292,7 @@ public class IncrementalIngester extends
   *@param list are the doc keys for the query.
   *@param componentHash is the component hash, if any, for the query.
   */
-  protected void getDocumentURIChunk(DeleteInfo[] rval, Map<String,Integer> map, String
outputConnectionName,
+  protected void getDocumentURIChunk(List<DeleteInfo> rval, String outputConnectionName,
     List<String> list, String componentHash)
     throws ManifoldCFException
   {
@@ -2329,19 +2309,15 @@ public class IncrementalIngester extends
     for (int i = 0; i < set.getRowCount(); i++)
     {
       IResultRow row = set.getRow(i);
-      String docHash = row.getValue(docKeyField).toString();
-      Integer position = (Integer)map.get(docHash);
-      if (position != null)
-      {
-        String lastURI = (String)row.getValue(docURIField);
-        if (lastURI != null && lastURI.length() == 0)
-          lastURI = null;
-        String lastURIHash = (String)row.getValue(uriHashField);
-        if (lastURIHash != null && lastURIHash.length() == 0)
-          lastURIHash = null;
-        String lastOutputVersion = (String)row.getValue(lastOutputVersionField);
-        rval[position.intValue()] = new DeleteInfo(lastURI,lastURIHash,lastOutputVersion);
-      }
+      //String docHash = row.getValue(docKeyField).toString();
+      String lastURI = (String)row.getValue(docURIField);
+      if (lastURI != null && lastURI.length() == 0)
+        lastURI = null;
+      String lastURIHash = (String)row.getValue(uriHashField);
+      if (lastURIHash != null && lastURIHash.length() == 0)
+        lastURIHash = null;
+      String lastOutputVersion = (String)row.getValue(lastOutputVersionField);
+      rval.add(new DeleteInfo(lastURI,lastURIHash,lastOutputVersion));
     }
   }
 



Mime
View raw message