manifoldcf-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kwri...@apache.org
Subject svn commit: r1792774 - in /manifoldcf/trunk: CHANGES.txt connectors/jdbc/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/jdbc/JDBCConnector.java
Date Wed, 26 Apr 2017 17:10:19 GMT
Author: kwright
Date: Wed Apr 26 17:10:19 2017
New Revision: 1792774

URL: http://svn.apache.org/viewvc?rev=1792774&view=rev
Log:
Fix for CONNECTORS-1419.

Modified:
    manifoldcf/trunk/CHANGES.txt
    manifoldcf/trunk/connectors/jdbc/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/jdbc/JDBCConnector.java

Modified: manifoldcf/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/CHANGES.txt?rev=1792774&r1=1792773&r2=1792774&view=diff
==============================================================================
--- manifoldcf/trunk/CHANGES.txt (original)
+++ manifoldcf/trunk/CHANGES.txt Wed Apr 26 17:10:19 2017
@@ -3,6 +3,10 @@ $Id$
 
 ======================= 2.8-dev =====================
 
+CONNECTORS-1419: Handle the JDBC no-version-query case so that
+deletes work.
+(Julien Massiera, Karl Wright)
+
 CONNECTORS-1417: Accept multipart/alternal email format.
 (Cihad Guzel)
 

Modified: manifoldcf/trunk/connectors/jdbc/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/jdbc/JDBCConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/jdbc/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/jdbc/JDBCConnector.java?rev=1792774&r1=1792773&r2=1792774&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/jdbc/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/jdbc/JDBCConnector.java
(original)
+++ manifoldcf/trunk/connectors/jdbc/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/jdbc/JDBCConnector.java
Wed Apr 26 17:10:19 2017
@@ -397,7 +397,7 @@ public class JDBCConnector extends org.a
     }
 
     // Delete the documents that had no version, and work only on ones that did
-    Set<String> fetchDocuments = documentVersions.keySet();
+    final Set<String> fetchDocuments = documentVersions.keySet();
     for (String documentIdentifier : documentIdentifiers)
     {
       String documentVersion = documentVersions.get(documentIdentifier);
@@ -497,7 +497,11 @@ public class JDBCConnector extends org.a
       }
     }
     
-    Map<String,String> map = new HashMap<String,String>();
+    // Map from identifier to version string
+    final Map<String,String> map = new HashMap<>();
+    // This is the set of documents actually seen
+    final Set<String> seenDocuments = new HashSet<>();
+    
     for (String documentIdentifier : fetchDocuments)
     {
       String documentVersion = documentVersions.get(documentIdentifier);
@@ -676,7 +680,8 @@ public class JDBCConnector extends org.a
           Object o = row.getValue(JDBCConstants.idReturnColumnName);
           if (o == null)
             throw new ManifoldCFException("Bad document query; doesn't return $(IDCOLUMN)
column.  Try using quotes around $(IDCOLUMN) variable, e.g. \"$(IDCOLUMN)\", or, for MySQL,
select \"by label\" in your repository connection.");
-          String id = JDBCConnection.readAsString(o);
+          final String id = JDBCConnection.readAsString(o);
+          seenDocuments.add(id);
           
           String errorCode = null;
           String errorDesc = null;
@@ -922,11 +927,17 @@ public class JDBCConnector extends org.a
     
     // Now, go through the original id's, and see which ones are still in the map.  These
     // did not appear in the result and are presumed to be gone from the database, and thus
must be deleted.
-    for (String documentIdentifier : documentIdentifiers)
+    for (final String documentIdentifier : fetchDocuments)
     {
-      if (fetchDocuments.contains(documentIdentifier))
+      if (!seenDocuments.contains(documentIdentifier))
+      {
+        // Never saw it in the fetch attempt
+        activities.deleteDocument(documentIdentifier);
+      }
+      else
       {
-        String documentVersion = map.get(documentIdentifier);
+        // Saw it in the fetch attempt, and we might have fetched it
+        final String documentVersion = map.get(documentIdentifier);
         if (documentVersion != null)
         {
           // This means we did not see it (or data for it) in the result set.  Delete it!



Mime
View raw message