manifoldcf-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kwri...@apache.org
Subject svn commit: r1611848 - /manifoldcf/trunk/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/HDFSRepositoryConnector.java
Date Sat, 19 Jul 2014 06:17:53 GMT
Author: kwright
Date: Sat Jul 19 06:17:52 2014
New Revision: 1611848

URL: http://svn.apache.org/r1611848
Log:
Add comments about versioning

Modified:
    manifoldcf/trunk/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/HDFSRepositoryConnector.java

Modified: manifoldcf/trunk/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/HDFSRepositoryConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/HDFSRepositoryConnector.java?rev=1611848&r1=1611847&r2=1611848&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/HDFSRepositoryConnector.java
(original)
+++ manifoldcf/trunk/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/HDFSRepositoryConnector.java
Sat Jul 19 06:17:52 2014
@@ -350,6 +350,10 @@ public class HDFSRepositoryConnector ext
       FileStatus fileStatus = getObject(new Path(documentIdentifier));
       if (fileStatus != null) {
         if (fileStatus.isDirectory()) {
+          // If HDFS directory modify dates are transitive, as they are on Unix,
+          // then getting the modify date of the current version is sufficient
+          // to detect any downstream changes we need to be aware of.
+          // (If this turns out to be a bad assumption, this should simply set rval[i] ="").
           long lastModified = fileStatus.getModificationTime();
           rval[i] = new Long(lastModified).toString();
         } else {
@@ -414,6 +418,8 @@ public class HDFSRepositoryConnector ext
       }
         
       if (fileStatus.isDirectory()) {
+        // Since we believe that downstream changes affect the current node's version string,
+        // then we only have to add references when there are detected changes.
         if (!scanOnly[i]) {
           activities.noDocument(documentIdentifier,version);
           /*



Mime
View raw message