manifoldcf-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kwri...@apache.org
Subject svn commit: r1370892 - in /manifoldcf/branches/CONNECTORS-501: framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/HopCount.java tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/HopcountTester.java
Date Wed, 08 Aug 2012 18:56:20 GMT
Author: kwright
Date: Wed Aug  8 18:56:20 2012
New Revision: 1370892

URL: http://svn.apache.org/viewvc?rev=1370892&view=rev
Log:
Revert the critical parts of CONNECTORS-464 in order to get the hopcount cutoff stuff working
correctly again.

Modified:
    manifoldcf/branches/CONNECTORS-501/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/HopCount.java
    manifoldcf/branches/CONNECTORS-501/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/HopcountTester.java

Modified: manifoldcf/branches/CONNECTORS-501/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/HopCount.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-501/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/HopCount.java?rev=1370892&r1=1370891&r2=1370892&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-501/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/HopCount.java
(original)
+++ manifoldcf/branches/CONNECTORS-501/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/HopCount.java
Wed Aug  8 18:56:20 2012
@@ -591,13 +591,11 @@ public class HopCount extends org.apache
       //
       // ... and then, re-evaluate all hopcount records and their dependencies that are marked
for delete.
       //
-      // But, the trick is that both source and target links must go away!!  So deleting
a document is very different than
-      // updating a link...
+
 
       // This also removes the links themselves...
       if (hopcountMethod == IJobDescription.HOPCOUNT_ACCURATE)
         doDeleteDocuments(jobID,documentHashes);
-      
 
     }
     catch (ManifoldCFException e)
@@ -1084,7 +1082,7 @@ public class HopCount extends org.apache
 
   }
 
-  /** Invalidate links that start with or end in a specific set of documents, described by
+  /** Invalidate links that start with a specific set of documents, described by
   * a table join.
   */
   protected void doDeleteDocuments(Long jobID,
@@ -1141,39 +1139,7 @@ public class HopCount extends org.apache
     performUpdate(map,sb.toString(),list,null);
     noteModifications(0,1,0);
       
-      
-    sb = new StringBuilder("WHERE ");
-    list = new ArrayList();
-        
-    sb.append(idField).append(" IN(SELECT t0.").append(deleteDepsManager.ownerIDField).append("
FROM ")
-      .append(deleteDepsManager.getTableName()).append(" t0,").append(joinTableName).append(",")
-      .append(intrinsicLinkManager.getTableName()).append(" t1 WHERE ");
-
-    sb.append(buildConjunctionClause(list,new ClauseDescription[]{
-      new UnitaryClause("t0."+deleteDepsManager.jobIDField,jobID)})).append(" AND ");
-
-    sb.append(buildConjunctionClause(list,new ClauseDescription[]{
-      new UnitaryClause("t1."+intrinsicLinkManager.jobIDField,jobID),
-      new JoinClause("t1."+intrinsicLinkManager.parentIDHashField,"t0."+deleteDepsManager.parentIDHashField),
-      new JoinClause("t1."+intrinsicLinkManager.linkTypeField,"t0."+deleteDepsManager.linkTypeField),
-      new JoinClause("t1."+intrinsicLinkManager.childIDHashField,"t0."+deleteDepsManager.childIDHashField)})).append("
AND ");
-
-    sb.append(buildConjunctionClause(list,new ClauseDescription[]{
-      new UnitaryClause(joinTableJobColumn,jobID),
-      new JoinClause(joinTableIDColumn,"t0."+deleteDepsManager.parentIDHashField)})).append("
AND ");
-          
-    sb.append(joinTableCriteria);
-    list.addAll(joinTableParams);
-
-    sb.append(")");
-
-    map = new HashMap();
-    // These are whacked back to "infinity" to avoid infinite looping in a cut-off graph.
-    map.put(distanceField,new Long(-1L));
-    map.put(markForDeathField,markToString(MARK_DELETING));
-    performUpdate(map,sb.toString(),list,null);
-    noteModifications(0,1,0);
-
+    // We do NOT do the parentID because otherwise we have the potential to delete links
that we need later.  See CONNECTORS-501.
 
     if (Logging.hopcount.isDebugEnabled())
       Logging.hopcount.debug("Done setting hopcount rows for job "+jobID+" to initial distances");
@@ -1206,7 +1172,7 @@ public class HopCount extends org.apache
 
   }
   
-  /** Invalidate links that start with or end in a specific set of documents.
+  /** Invalidate links that start with a specific set of documents.
   */
   protected void doDeleteDocuments(Long jobID,
     String[] documentHashes)
@@ -1327,28 +1293,8 @@ public class HopCount extends org.apache
     map.put(markForDeathField,markToString(MARK_DELETING));
     performUpdate(map,sb.toString(),thisList,null);
 
-    sb = new StringBuilder("WHERE ");
-    thisList = new ArrayList();
-
-    sb.append(idField).append(" IN(SELECT ").append(deleteDepsManager.ownerIDField).append("
FROM ").append(deleteDepsManager.getTableName()).append(" t0 WHERE ")
-      .append(buildConjunctionClause(thisList,new ClauseDescription[]{
-        new UnitaryClause("t0."+deleteDepsManager.jobIDField,jobID),
-        new MultiClause("t0."+deleteDepsManager.parentIDHashField,list)})).append(" AND ");
-        
-    sb.append("EXISTS(SELECT 'x' FROM ").append(intrinsicLinkManager.getTableName()).append("
t1 WHERE ")
-      .append(buildConjunctionClause(thisList,new ClauseDescription[]{
-        new JoinClause("t1."+intrinsicLinkManager.jobIDField,"t0."+deleteDepsManager.jobIDField),
-        new JoinClause("t1."+intrinsicLinkManager.linkTypeField,"t0."+deleteDepsManager.linkTypeField),
-        new JoinClause("t1."+intrinsicLinkManager.parentIDHashField,"t0."+deleteDepsManager.parentIDHashField),
-        new JoinClause("t1."+intrinsicLinkManager.childIDHashField,"t0."+deleteDepsManager.childIDHashField)}));
-
-    sb.append("))");
-        
-    map = new HashMap();
-    // These are whacked back to "infinity" to avoid infinite looping in a cut-off graph.
-    map.put(distanceField,new Long(-1L));
-    map.put(markForDeathField,markToString(MARK_DELETING));
-    performUpdate(map,sb.toString(),thisList,null);
+    // We do NOT do the parentID because we need to leave intrinsic links around that could
be used again.
+    // See CONNECTORS-501.
   }
 
   /** Invalidate links meeting a simple criteria which have a given set of source documents.
 This also runs a queue

Modified: manifoldcf/branches/CONNECTORS-501/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/HopcountTester.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-501/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/HopcountTester.java?rev=1370892&r1=1370891&r2=1370892&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-501/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/HopcountTester.java
(original)
+++ manifoldcf/branches/CONNECTORS-501/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/HopcountTester.java
Wed Aug  8 18:56:20 2012
@@ -134,7 +134,7 @@ public class HopcountTester
     // But the max hopcount is 2, so one file will be left behind, so the count should be
6, not 7.
     if (status.getDocumentsProcessed() != 6)
       throw new ManifoldCFException("Wrong number of documents processed - expected 6, saw
"+new Long(status.getDocumentsProcessed()).toString());
-
+      
     // Now, delete the job.
     jobManager.deleteJob(job.getID());
     instance.waitJobDeletedNative(jobManager,job.getID(), 120000L);



Mime
View raw message