manifoldcf-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kwri...@apache.org
Subject svn commit: r1614295 - in /manifoldcf/trunk/connectors: hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/ livelink/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/livelink/
Date Tue, 29 Jul 2014 09:42:49 GMT
Author: kwright
Date: Tue Jul 29 09:42:49 2014
New Revision: 1614295

URL: http://svn.apache.org/r1614295
Log:
Remove versioning for folders for HDFS and Livelink: HDFS because it emulates unix and windows,
and Livelink because we don't know and won't know in time for 1.7

Modified:
    manifoldcf/trunk/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/HDFSRepositoryConnector.java
    manifoldcf/trunk/connectors/livelink/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/livelink/LivelinkConnector.java

Modified: manifoldcf/trunk/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/HDFSRepositoryConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/HDFSRepositoryConnector.java?rev=1614295&r1=1614294&r2=1614295&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/HDFSRepositoryConnector.java
(original)
+++ manifoldcf/trunk/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/HDFSRepositoryConnector.java
Tue Jul 29 09:42:49 2014
@@ -418,26 +418,19 @@ public class HDFSRepositoryConnector ext
       }
         
       if (fileStatus.isDirectory()) {
-        // Since we believe that downstream changes affect the current node's version string,
-        // then we only have to add references when there are detected changes.
-        if (!scanOnly[i]) {
-          activities.noDocument(documentIdentifier,version);
-          /*
-            * Queue up stuff for directory
-            */
-          String entityReference = documentIdentifier;
-          FileStatus[] fileStatuses = getChildren(fileStatus.getPath());
-          if (fileStatuses == null) {
-            // Directory was deleted, so remove
-            activities.deleteDocument(documentIdentifier);
-            continue;
-          }
-          for (int j = 0; j < fileStatuses.length; j++) {
-            FileStatus fs = fileStatuses[j++];
-            String canonicalPath = fs.getPath().toString();
-            if (checkInclude(session.getUri().toString(),fs,canonicalPath,spec)) {
-              activities.addDocumentReference(canonicalPath,documentIdentifier,RELATIONSHIP_CHILD);
-            }
+        /*
+          * Queue up stuff for directory
+          */
+        String entityReference = documentIdentifier;
+        FileStatus[] fileStatuses = getChildren(fileStatus.getPath());
+        if (fileStatuses == null) {
+          continue;
+        }
+        for (int j = 0; j < fileStatuses.length; j++) {
+          FileStatus fs = fileStatuses[j++];
+          String canonicalPath = fs.getPath().toString();
+          if (checkInclude(session.getUri().toString(),fs,canonicalPath,spec)) {
+            activities.addDocumentReference(canonicalPath,documentIdentifier,RELATIONSHIP_CHILD);
           }
         }
       } else {

Modified: manifoldcf/trunk/connectors/livelink/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/livelink/LivelinkConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/livelink/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/livelink/LivelinkConnector.java?rev=1614295&r1=1614294&r2=1614295&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/livelink/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/livelink/LivelinkConnector.java
(original)
+++ manifoldcf/trunk/connectors/livelink/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/livelink/LivelinkConnector.java
Tue Jul 29 09:42:49 2014
@@ -1473,103 +1473,99 @@ public class LivelinkConnector extends o
 
       if (isFolder)
       {
-        if (doScanOnly == false)
-        {
-          activities.noDocument(documentIdentifier,version);
-          if (Logging.connectors.isDebugEnabled())
-            Logging.connectors.debug("Livelink: Processing folder "+Integer.toString(vol)+":"+Integer.toString(objID));
+        // We don't know if LL will let us version folders properly, so ALWAYS process folders.
+        if (Logging.connectors.isDebugEnabled())
+          Logging.connectors.debug("Livelink: Processing folder "+Integer.toString(vol)+":"+Integer.toString(objID));
 
-          // Since the identifier indicates it is a directory, then queue up all the current
children which pass the filter.
-          String filterString = buildFilterString(spec);
+        // Since the identifier indicates it is a directory, then queue up all the current
children which pass the filter.
+        String filterString = buildFilterString(spec);
 
-          int sanityRetryCount = FAILURE_RETRY_COUNT;
-          while (true)
+        int sanityRetryCount = FAILURE_RETRY_COUNT;
+        while (true)
+        {
+          ListObjectsThread t = new ListObjectsThread(vol,objID,filterString);
+          try
           {
-            ListObjectsThread t = new ListObjectsThread(vol,objID,filterString);
+            t.start();
+            LLValue childrenDocs;
             try
             {
-              t.start();
-              LLValue childrenDocs;
-              try
-              {
-                childrenDocs = t.finishUp();
-              }
-              catch (ManifoldCFException e)
-              {
-                sanityRetryCount = assessRetry(sanityRetryCount,e);
-                continue;
-              }
+              childrenDocs = t.finishUp();
+            }
+            catch (ManifoldCFException e)
+            {
+              sanityRetryCount = assessRetry(sanityRetryCount,e);
+              continue;
+            }
 
-              int size = 0;
+            int size = 0;
+            
+            if (childrenDocs.isRecord())
+              size = 1;
+            if (childrenDocs.isTable())
+              size = childrenDocs.size();
 
-              if (childrenDocs.isRecord())
-                size = 1;
-              if (childrenDocs.isTable())
-                size = childrenDocs.size();
-
-              // System.out.println("Total child count = "+Integer.toString(size));
-
-              // Do the scan
-              int j = 0;
-              while (j < size)
-              {
-                int childID = childrenDocs.toInteger(j, "ID");
+            // System.out.println("Total child count = "+Integer.toString(size));
+            // Do the scan
+            int j = 0;
+            while (j < size)
+            {
+              int childID = childrenDocs.toInteger(j, "ID");
 
-                if (Logging.connectors.isDebugEnabled())
-                  Logging.connectors.debug("Livelink: Found a child of folder "+Integer.toString(vol)+":"+Integer.toString(objID)+"
: ID="+Integer.toString(childID));
+              if (Logging.connectors.isDebugEnabled())
+                Logging.connectors.debug("Livelink: Found a child of folder "+Integer.toString(vol)+":"+Integer.toString(objID)+"
: ID="+Integer.toString(childID));
 
-                int subtype = childrenDocs.toInteger(j, "SubType");
-                boolean childIsFolder = (subtype == LAPI_DOCUMENTS.FOLDERSUBTYPE || subtype
== LAPI_DOCUMENTS.PROJECTSUBTYPE ||
-                  subtype == LAPI_DOCUMENTS.COMPOUNDDOCUMENTSUBTYPE);
+              int subtype = childrenDocs.toInteger(j, "SubType");
+              boolean childIsFolder = (subtype == LAPI_DOCUMENTS.FOLDERSUBTYPE || subtype
== LAPI_DOCUMENTS.PROJECTSUBTYPE ||
+                subtype == LAPI_DOCUMENTS.COMPOUNDDOCUMENTSUBTYPE);
 
-                // If it's a folder, we just let it through for now
-                if (!childIsFolder && checkInclude(childrenDocs.toString(j,"Name")
+ "." + childrenDocs.toString(j,"FileType"), spec) == false)
-                {
-                  if (Logging.connectors.isDebugEnabled())
-                    Logging.connectors.debug("Livelink: Child identifier "+Integer.toString(childID)+"
was excluded by inclusion criteria");
-                  j++;
-                  continue;
-                }
+              // If it's a folder, we just let it through for now
+              if (!childIsFolder && checkInclude(childrenDocs.toString(j,"Name")
+ "." + childrenDocs.toString(j,"FileType"), spec) == false)
+              {
+                if (Logging.connectors.isDebugEnabled())
+                  Logging.connectors.debug("Livelink: Child identifier "+Integer.toString(childID)+"
was excluded by inclusion criteria");
+                j++;
+                continue;
+              }
 
-                if (childIsFolder)
+              if (childIsFolder)
+              {
+                if (Logging.connectors.isDebugEnabled())
+                  Logging.connectors.debug("Livelink: Child identifier "+Integer.toString(childID)+"
is a folder, project, or compound document; adding a reference");
+                if (subtype == LAPI_DOCUMENTS.PROJECTSUBTYPE)
                 {
-                  if (Logging.connectors.isDebugEnabled())
-                    Logging.connectors.debug("Livelink: Child identifier "+Integer.toString(childID)+"
is a folder, project, or compound document; adding a reference");
-                  if (subtype == LAPI_DOCUMENTS.PROJECTSUBTYPE)
-                  {
-                    // If we pick up a project object, we need to describe the volume object
(which
-                    // will be the root of all documents beneath)
-                    activities.addDocumentReference("F"+new Integer(childID).toString()+":"+new
Integer(-childID).toString());
-                  }
-                  else
-                    activities.addDocumentReference("F"+new Integer(vol).toString()+":"+new
Integer(childID).toString());
+                  // If we pick up a project object, we need to describe the volume object
(which
+                  // will be the root of all documents beneath)
+                  activities.addDocumentReference("F"+new Integer(childID).toString()+":"+new
Integer(-childID).toString());
                 }
                 else
-                {
-                  if (Logging.connectors.isDebugEnabled())
-                    Logging.connectors.debug("Livelink: Child identifier "+Integer.toString(childID)+"
is a simple document; adding a reference");
-
-                  activities.addDocumentReference("D"+new Integer(vol).toString()+":"+new
Integer(childID).toString());
-                }
+                  activities.addDocumentReference("F"+new Integer(vol).toString()+":"+new
Integer(childID).toString());
+              }
+              else
+              {
+                if (Logging.connectors.isDebugEnabled())
+                  Logging.connectors.debug("Livelink: Child identifier "+Integer.toString(childID)+"
is a simple document; adding a reference");
 
-                j++;
+                activities.addDocumentReference("D"+new Integer(vol).toString()+":"+new Integer(childID).toString());
               }
-              break;
-            }
-            catch (InterruptedException e)
-            {
-              t.interrupt();
-              throw new ManifoldCFException("Interrupted: "+e.getMessage(),e,ManifoldCFException.INTERRUPTED);
-            }
-            catch (RuntimeException e)
-            {
-              sanityRetryCount = handleLivelinkRuntimeException(e,sanityRetryCount,true);
-              continue;
+
+              j++;
             }
+            break;
+          }
+          catch (InterruptedException e)
+          {
+            t.interrupt();
+            throw new ManifoldCFException("Interrupted: "+e.getMessage(),e,ManifoldCFException.INTERRUPTED);
+          }
+          catch (RuntimeException e)
+          {
+            sanityRetryCount = handleLivelinkRuntimeException(e,sanityRetryCount,true);
+            continue;
           }
-          if (Logging.connectors.isDebugEnabled())
-            Logging.connectors.debug("Livelink: Done processing folder "+Integer.toString(vol)+":"+Integer.toString(objID));
         }
+        if (Logging.connectors.isDebugEnabled())
+          Logging.connectors.debug("Livelink: Done processing folder "+Integer.toString(vol)+":"+Integer.toString(objID));
       }
       else
       {



Mime
View raw message