Return-Path: X-Original-To: apmail-manifoldcf-commits-archive@www.apache.org Delivered-To: apmail-manifoldcf-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 0E6E111917 for ; Wed, 16 Jul 2014 14:05:17 +0000 (UTC) Received: (qmail 34914 invoked by uid 500); 16 Jul 2014 14:05:16 -0000 Delivered-To: apmail-manifoldcf-commits-archive@manifoldcf.apache.org Received: (qmail 34860 invoked by uid 500); 16 Jul 2014 14:05:16 -0000 Mailing-List: contact commits-help@manifoldcf.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@manifoldcf.apache.org Delivered-To: mailing list commits@manifoldcf.apache.org Received: (qmail 34849 invoked by uid 99); 16 Jul 2014 14:05:16 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 16 Jul 2014 14:05:16 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 16 Jul 2014 14:05:15 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id ADCE62388831; Wed, 16 Jul 2014 14:04:48 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1611015 - in /manifoldcf/trunk/connectors: filesystem/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/filesystem/ hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/ livelink/connector/src/main/java... Date: Wed, 16 Jul 2014 14:04:48 -0000 To: commits@manifoldcf.apache.org From: kwright@apache.org X-Mailer: svnmailer-1.0.9 Message-Id: <20140716140448.ADCE62388831@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: kwright Date: Wed Jul 16 14:04:47 2014 New Revision: 1611015 URL: http://svn.apache.org/r1611015 Log: Also pay attention to scanonly flag; part of CONNECTORS-995 Modified: manifoldcf/trunk/connectors/filesystem/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/filesystem/FileConnector.java manifoldcf/trunk/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/HDFSRepositoryConnector.java manifoldcf/trunk/connectors/livelink/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/livelink/LivelinkConnector.java manifoldcf/trunk/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/sharepoint/SharePointRepository.java Modified: manifoldcf/trunk/connectors/filesystem/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/filesystem/FileConnector.java URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/filesystem/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/filesystem/FileConnector.java?rev=1611015&r1=1611014&r2=1611015&view=diff ============================================================================== --- manifoldcf/trunk/connectors/filesystem/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/filesystem/FileConnector.java (original) +++ manifoldcf/trunk/connectors/filesystem/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/filesystem/FileConnector.java Wed Jul 16 14:04:47 2014 @@ -289,40 +289,43 @@ public class FileConnector extends org.a { if (file.isDirectory()) { - activities.noDocument(documentIdentifier,version); - // Queue up stuff for directory - long startTime = System.currentTimeMillis(); - String errorCode = "OK"; - String errorDesc = null; - String entityReference = documentIdentifier; - try + if (!scanOnly[i]) { + activities.noDocument(documentIdentifier,version); + // Queue up stuff for directory + long startTime = System.currentTimeMillis(); + String errorCode = "OK"; + String errorDesc = null; + String entityReference = documentIdentifier; try { - File[] files = file.listFiles(); - if (files != null) + try { - int j = 0; - while (j < files.length) + File[] files = file.listFiles(); + if (files != null) { - File f = files[j++]; - String canonicalPath = f.getCanonicalPath(); - if (checkInclude(f,canonicalPath,spec)) - activities.addDocumentReference(canonicalPath,documentIdentifier,RELATIONSHIP_CHILD); + int j = 0; + while (j < files.length) + { + File f = files[j++]; + String canonicalPath = f.getCanonicalPath(); + if (checkInclude(f,canonicalPath,spec)) + activities.addDocumentReference(canonicalPath,documentIdentifier,RELATIONSHIP_CHILD); + } } } + catch (IOException e) + { + errorCode = "IO ERROR"; + errorDesc = e.getMessage(); + throw new ManifoldCFException("IO Error: "+e.getMessage(),e); + } } - catch (IOException e) + finally { - errorCode = "IO ERROR"; - errorDesc = e.getMessage(); - throw new ManifoldCFException("IO Error: "+e.getMessage(),e); + activities.recordActivity(new Long(startTime),ACTIVITY_READ,null,entityReference,errorCode,errorDesc,null); } } - finally - { - activities.recordActivity(new Long(startTime),ACTIVITY_READ,null,entityReference,errorCode,errorDesc,null); - } } else { Modified: manifoldcf/trunk/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/HDFSRepositoryConnector.java URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/HDFSRepositoryConnector.java?rev=1611015&r1=1611014&r2=1611015&view=diff ============================================================================== --- manifoldcf/trunk/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/HDFSRepositoryConnector.java (original) +++ manifoldcf/trunk/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/HDFSRepositoryConnector.java Wed Jul 16 14:04:47 2014 @@ -414,22 +414,24 @@ public class HDFSRepositoryConnector ext } if (fileStatus.isDirectory()) { - activities.noDocument(documentIdentifier,version); - /* - * Queue up stuff for directory - */ - String entityReference = documentIdentifier; - FileStatus[] fileStatuses = getChildren(fileStatus.getPath()); - if (fileStatuses == null) { - // Directory was deleted, so remove - activities.deleteDocument(documentIdentifier); - continue; - } - for (int j = 0; j < fileStatuses.length; j++) { - FileStatus fs = fileStatuses[j++]; - String canonicalPath = fs.getPath().toString(); - if (checkInclude(session.getUri().toString(),fs,canonicalPath,spec)) { - activities.addDocumentReference(canonicalPath,documentIdentifier,RELATIONSHIP_CHILD); + if (!scanOnly[i]) { + activities.noDocument(documentIdentifier,version); + /* + * Queue up stuff for directory + */ + String entityReference = documentIdentifier; + FileStatus[] fileStatuses = getChildren(fileStatus.getPath()); + if (fileStatuses == null) { + // Directory was deleted, so remove + activities.deleteDocument(documentIdentifier); + continue; + } + for (int j = 0; j < fileStatuses.length; j++) { + FileStatus fs = fileStatuses[j++]; + String canonicalPath = fs.getPath().toString(); + if (checkInclude(session.getUri().toString(),fs,canonicalPath,spec)) { + activities.addDocumentReference(canonicalPath,documentIdentifier,RELATIONSHIP_CHILD); + } } } } else { Modified: manifoldcf/trunk/connectors/livelink/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/livelink/LivelinkConnector.java URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/livelink/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/livelink/LivelinkConnector.java?rev=1611015&r1=1611014&r2=1611015&view=diff ============================================================================== --- manifoldcf/trunk/connectors/livelink/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/livelink/LivelinkConnector.java (original) +++ manifoldcf/trunk/connectors/livelink/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/livelink/LivelinkConnector.java Wed Jul 16 14:04:47 2014 @@ -1473,100 +1473,103 @@ public class LivelinkConnector extends o if (isFolder) { - activities.noDocument(documentIdentifier,version); - if (Logging.connectors.isDebugEnabled()) - Logging.connectors.debug("Livelink: Processing folder "+Integer.toString(vol)+":"+Integer.toString(objID)); - - // Since the identifier indicates it is a directory, then queue up all the current children which pass the filter. - String filterString = buildFilterString(spec); - - int sanityRetryCount = FAILURE_RETRY_COUNT; - while (true) + if (doScanOnly == false) { - ListObjectsThread t = new ListObjectsThread(vol,objID,filterString); - try - { - t.start(); - LLValue childrenDocs; - try - { - childrenDocs = t.finishUp(); - } - catch (ManifoldCFException e) - { - sanityRetryCount = assessRetry(sanityRetryCount,e); - continue; - } - - int size = 0; - - if (childrenDocs.isRecord()) - size = 1; - if (childrenDocs.isTable()) - size = childrenDocs.size(); + activities.noDocument(documentIdentifier,version); + if (Logging.connectors.isDebugEnabled()) + Logging.connectors.debug("Livelink: Processing folder "+Integer.toString(vol)+":"+Integer.toString(objID)); - // System.out.println("Total child count = "+Integer.toString(size)); + // Since the identifier indicates it is a directory, then queue up all the current children which pass the filter. + String filterString = buildFilterString(spec); - // Do the scan - int j = 0; - while (j < size) + int sanityRetryCount = FAILURE_RETRY_COUNT; + while (true) + { + ListObjectsThread t = new ListObjectsThread(vol,objID,filterString); + try { - int childID = childrenDocs.toInteger(j, "ID"); - - if (Logging.connectors.isDebugEnabled()) - Logging.connectors.debug("Livelink: Found a child of folder "+Integer.toString(vol)+":"+Integer.toString(objID)+" : ID="+Integer.toString(childID)); - - int subtype = childrenDocs.toInteger(j, "SubType"); - boolean childIsFolder = (subtype == LAPI_DOCUMENTS.FOLDERSUBTYPE || subtype == LAPI_DOCUMENTS.PROJECTSUBTYPE || - subtype == LAPI_DOCUMENTS.COMPOUNDDOCUMENTSUBTYPE); - - // If it's a folder, we just let it through for now - if (!childIsFolder && checkInclude(childrenDocs.toString(j,"Name") + "." + childrenDocs.toString(j,"FileType"), spec) == false) + t.start(); + LLValue childrenDocs; + try { - if (Logging.connectors.isDebugEnabled()) - Logging.connectors.debug("Livelink: Child identifier "+Integer.toString(childID)+" was excluded by inclusion criteria"); - j++; + childrenDocs = t.finishUp(); + } + catch (ManifoldCFException e) + { + sanityRetryCount = assessRetry(sanityRetryCount,e); continue; } - if (childIsFolder) + int size = 0; + + if (childrenDocs.isRecord()) + size = 1; + if (childrenDocs.isTable()) + size = childrenDocs.size(); + + // System.out.println("Total child count = "+Integer.toString(size)); + + // Do the scan + int j = 0; + while (j < size) { + int childID = childrenDocs.toInteger(j, "ID"); + if (Logging.connectors.isDebugEnabled()) - Logging.connectors.debug("Livelink: Child identifier "+Integer.toString(childID)+" is a folder, project, or compound document; adding a reference"); - if (subtype == LAPI_DOCUMENTS.PROJECTSUBTYPE) + Logging.connectors.debug("Livelink: Found a child of folder "+Integer.toString(vol)+":"+Integer.toString(objID)+" : ID="+Integer.toString(childID)); + + int subtype = childrenDocs.toInteger(j, "SubType"); + boolean childIsFolder = (subtype == LAPI_DOCUMENTS.FOLDERSUBTYPE || subtype == LAPI_DOCUMENTS.PROJECTSUBTYPE || + subtype == LAPI_DOCUMENTS.COMPOUNDDOCUMENTSUBTYPE); + + // If it's a folder, we just let it through for now + if (!childIsFolder && checkInclude(childrenDocs.toString(j,"Name") + "." + childrenDocs.toString(j,"FileType"), spec) == false) { - // If we pick up a project object, we need to describe the volume object (which - // will be the root of all documents beneath) - activities.addDocumentReference("F"+new Integer(childID).toString()+":"+new Integer(-childID).toString()); + if (Logging.connectors.isDebugEnabled()) + Logging.connectors.debug("Livelink: Child identifier "+Integer.toString(childID)+" was excluded by inclusion criteria"); + j++; + continue; + } + + if (childIsFolder) + { + if (Logging.connectors.isDebugEnabled()) + Logging.connectors.debug("Livelink: Child identifier "+Integer.toString(childID)+" is a folder, project, or compound document; adding a reference"); + if (subtype == LAPI_DOCUMENTS.PROJECTSUBTYPE) + { + // If we pick up a project object, we need to describe the volume object (which + // will be the root of all documents beneath) + activities.addDocumentReference("F"+new Integer(childID).toString()+":"+new Integer(-childID).toString()); + } + else + activities.addDocumentReference("F"+new Integer(vol).toString()+":"+new Integer(childID).toString()); } else - activities.addDocumentReference("F"+new Integer(vol).toString()+":"+new Integer(childID).toString()); - } - else - { - if (Logging.connectors.isDebugEnabled()) - Logging.connectors.debug("Livelink: Child identifier "+Integer.toString(childID)+" is a simple document; adding a reference"); + { + if (Logging.connectors.isDebugEnabled()) + Logging.connectors.debug("Livelink: Child identifier "+Integer.toString(childID)+" is a simple document; adding a reference"); - activities.addDocumentReference("D"+new Integer(vol).toString()+":"+new Integer(childID).toString()); - } + activities.addDocumentReference("D"+new Integer(vol).toString()+":"+new Integer(childID).toString()); + } - j++; + j++; + } + break; + } + catch (InterruptedException e) + { + t.interrupt(); + throw new ManifoldCFException("Interrupted: "+e.getMessage(),e,ManifoldCFException.INTERRUPTED); + } + catch (RuntimeException e) + { + sanityRetryCount = handleLivelinkRuntimeException(e,sanityRetryCount,true); + continue; } - break; - } - catch (InterruptedException e) - { - t.interrupt(); - throw new ManifoldCFException("Interrupted: "+e.getMessage(),e,ManifoldCFException.INTERRUPTED); - } - catch (RuntimeException e) - { - sanityRetryCount = handleLivelinkRuntimeException(e,sanityRetryCount,true); - continue; } + if (Logging.connectors.isDebugEnabled()) + Logging.connectors.debug("Livelink: Done processing folder "+Integer.toString(vol)+":"+Integer.toString(objID)); } - if (Logging.connectors.isDebugEnabled()) - Logging.connectors.debug("Livelink: Done processing folder "+Integer.toString(vol)+":"+Integer.toString(objID)); } else { Modified: manifoldcf/trunk/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/sharepoint/SharePointRepository.java URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/sharepoint/SharePointRepository.java?rev=1611015&r1=1611014&r2=1611015&view=diff ============================================================================== --- manifoldcf/trunk/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/sharepoint/SharePointRepository.java (original) +++ manifoldcf/trunk/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/sharepoint/SharePointRepository.java Wed Jul 16 14:04:47 2014 @@ -1266,87 +1266,90 @@ public class SharePointRepository extend // === List style identifier === if (dListSeparatorIndex == documentIdentifier.length() - 3) { - activities.noDocument(documentIdentifier,version); + if (!scanOnly[i]) + { + activities.noDocument(documentIdentifier,version); - String siteListPath = documentIdentifier.substring(0,documentIdentifier.length()-3); - int listCutoff = siteListPath.lastIndexOf( "/" ); - String site = siteListPath.substring(0,listCutoff); - String listName = siteListPath.substring( listCutoff + 1 ); + String siteListPath = documentIdentifier.substring(0,documentIdentifier.length()-3); + int listCutoff = siteListPath.lastIndexOf( "/" ); + String site = siteListPath.substring(0,listCutoff); + String listName = siteListPath.substring( listCutoff + 1 ); - if (Logging.connectors.isDebugEnabled()) - Logging.connectors.debug( "SharePoint: Document identifier is a list: '" + siteListPath + "'" ); + if (Logging.connectors.isDebugEnabled()) + Logging.connectors.debug( "SharePoint: Document identifier is a list: '" + siteListPath + "'" ); - String listID = proxy.getListID( encodePath(site), site, listName ); - if (listID != null) - { - String encodedSitePath = encodePath(site); - - // Get the list's fields - Map fieldNames = proxy.getFieldList( encodedSitePath, listID ); - if (fieldNames != null) - { - String[] fields = new String[fieldNames.size()]; - int j = 0; - for (String field : fieldNames.keySet()) - { - fields[j++] = field; - } - - String[] accessTokens; - String[] denyTokens; + String listID = proxy.getListID( encodePath(site), site, listName ); + if (listID != null) + { + String encodedSitePath = encodePath(site); - if (forcedAcls == null) - { - // Security is off - accessTokens = new String[0]; - denyTokens = new String[0]; - } - else if (forcedAcls.length != 0) - { - // Forced security - accessTokens = forcedAcls; - denyTokens = new String[0]; - } - else - { - // Security enabled, native security - accessTokens = proxy.getACLs( encodedSitePath, listID, activeDirectoryAuthority ); - denyTokens = new String[]{defaultAuthorityDenyToken}; - } + // Get the list's fields + Map fieldNames = proxy.getFieldList( encodedSitePath, listID ); + if (fieldNames != null) + { + String[] fields = new String[fieldNames.size()]; + int j = 0; + for (String field : fieldNames.keySet()) + { + fields[j++] = field; + } + + String[] accessTokens; + String[] denyTokens; + + if (forcedAcls == null) + { + // Security is off + accessTokens = new String[0]; + denyTokens = new String[0]; + } + else if (forcedAcls.length != 0) + { + // Forced security + accessTokens = forcedAcls; + denyTokens = new String[0]; + } + else + { + // Security enabled, native security + accessTokens = proxy.getACLs( encodedSitePath, listID, activeDirectoryAuthority ); + denyTokens = new String[]{defaultAuthorityDenyToken}; + } - if (accessTokens != null) - { - ListItemStream fs = new ListItemStream( activities, encodedServerLocation, site, siteListPath, spec, - documentIdentifier, accessTokens, denyTokens, listID, fields ); - boolean success = proxy.getChildren( fs, encodedSitePath , listID, dspStsWorks ); - if (!success) + if (accessTokens != null) + { + ListItemStream fs = new ListItemStream( activities, encodedServerLocation, site, siteListPath, spec, + documentIdentifier, accessTokens, denyTokens, listID, fields ); + boolean success = proxy.getChildren( fs, encodedSitePath , listID, dspStsWorks ); + if (!success) + { + // Site/list no longer exists, so delete entry + if (Logging.connectors.isDebugEnabled()) + Logging.connectors.debug("SharePoint: No list found for list '"+siteListPath+"' - deleting"); + activities.deleteDocument(documentIdentifier); + } + } + else { - // Site/list no longer exists, so delete entry if (Logging.connectors.isDebugEnabled()) - Logging.connectors.debug("SharePoint: No list found for list '"+siteListPath+"' - deleting"); - activities.deleteDocument(documentIdentifier); + Logging.connectors.debug("SharePoint: Access token lookup failed for list '"+siteListPath+"' - deleting"); + activities.noDocument(documentIdentifier,version); } } else { if (Logging.connectors.isDebugEnabled()) - Logging.connectors.debug("SharePoint: Access token lookup failed for list '"+siteListPath+"' - deleting"); + Logging.connectors.debug("SharePoint: Field list lookup failed for list '"+siteListPath+"' - deleting"); activities.noDocument(documentIdentifier,version); } } else { if (Logging.connectors.isDebugEnabled()) - Logging.connectors.debug("SharePoint: Field list lookup failed for list '"+siteListPath+"' - deleting"); + Logging.connectors.debug("SharePoint: GUID lookup failed for list '"+siteListPath+"' - deleting"); activities.noDocument(documentIdentifier,version); } } - else - { - if (Logging.connectors.isDebugEnabled()) - Logging.connectors.debug("SharePoint: GUID lookup failed for list '"+siteListPath+"' - deleting"); - activities.noDocument(documentIdentifier,version); - } } else { @@ -1609,88 +1612,91 @@ public class SharePointRepository extend // === Library style identifier === if (dLibSeparatorIndex == documentIdentifier.length() - 2) { - // It's a library. - activities.noDocument(documentIdentifier,version); + if (!scanOnly[i]) + { + // It's a library. + activities.noDocument(documentIdentifier,version); - String siteLibPath = documentIdentifier.substring(0,documentIdentifier.length()-2); - int libCutoff = siteLibPath.lastIndexOf( "/" ); - String site = siteLibPath.substring(0,libCutoff); - String libName = siteLibPath.substring( libCutoff + 1 ); + String siteLibPath = documentIdentifier.substring(0,documentIdentifier.length()-2); + int libCutoff = siteLibPath.lastIndexOf( "/" ); + String site = siteLibPath.substring(0,libCutoff); + String libName = siteLibPath.substring( libCutoff + 1 ); - if (Logging.connectors.isDebugEnabled()) - Logging.connectors.debug( "SharePoint: Document identifier is a library: '" + siteLibPath + "'" ); + if (Logging.connectors.isDebugEnabled()) + Logging.connectors.debug( "SharePoint: Document identifier is a library: '" + siteLibPath + "'" ); - String libID = proxy.getDocLibID( encodePath(site), site, libName ); - if (libID != null) - { - String encodedSitePath = encodePath(site); - - // Get the lib's fields - Map fieldNames = proxy.getFieldList( encodedSitePath, libID ); - if (fieldNames != null) - { - String[] fields = new String[fieldNames.size()]; - int j = 0; - for (String field : fieldNames.keySet()) - { - fields[j++] = field; - } - - String[] accessTokens; - String[] denyTokens; + String libID = proxy.getDocLibID( encodePath(site), site, libName ); + if (libID != null) + { + String encodedSitePath = encodePath(site); - if (forcedAcls == null) - { - // Security is off - accessTokens = new String[0]; - denyTokens = new String[0]; - } - else if (forcedAcls.length != 0) - { - // Forced security - accessTokens = forcedAcls; - denyTokens = new String[0]; - } - else - { - // Security enabled, native security - accessTokens = proxy.getACLs( encodedSitePath, libID, activeDirectoryAuthority ); - denyTokens = new String[]{defaultAuthorityDenyToken}; - } + // Get the lib's fields + Map fieldNames = proxy.getFieldList( encodedSitePath, libID ); + if (fieldNames != null) + { + String[] fields = new String[fieldNames.size()]; + int j = 0; + for (String field : fieldNames.keySet()) + { + fields[j++] = field; + } + + String[] accessTokens; + String[] denyTokens; + + if (forcedAcls == null) + { + // Security is off + accessTokens = new String[0]; + denyTokens = new String[0]; + } + else if (forcedAcls.length != 0) + { + // Forced security + accessTokens = forcedAcls; + denyTokens = new String[0]; + } + else + { + // Security enabled, native security + accessTokens = proxy.getACLs( encodedSitePath, libID, activeDirectoryAuthority ); + denyTokens = new String[]{defaultAuthorityDenyToken}; + } - if (accessTokens != null) - { - FileStream fs = new FileStream( activities, encodedServerLocation, site, siteLibPath, spec, - documentIdentifier, accessTokens, denyTokens, libID, fields ); - boolean success = proxy.getChildren( fs, encodedSitePath , libID, dspStsWorks ); - if (!success) + if (accessTokens != null) + { + FileStream fs = new FileStream( activities, encodedServerLocation, site, siteLibPath, spec, + documentIdentifier, accessTokens, denyTokens, libID, fields ); + boolean success = proxy.getChildren( fs, encodedSitePath , libID, dspStsWorks ); + if (!success) + { + // Site/library no longer exists, so delete entry + if (Logging.connectors.isDebugEnabled()) + Logging.connectors.debug("SharePoint: No list found for library '"+siteLibPath+"' - deleting"); + activities.deleteDocument(documentIdentifier); + } + } + else { - // Site/library no longer exists, so delete entry if (Logging.connectors.isDebugEnabled()) - Logging.connectors.debug("SharePoint: No list found for library '"+siteLibPath+"' - deleting"); - activities.deleteDocument(documentIdentifier); + Logging.connectors.debug("SharePoint: Access token lookup failed for library '"+siteLibPath+"' - deleting"); + activities.noDocument(documentIdentifier,version); } } else { if (Logging.connectors.isDebugEnabled()) - Logging.connectors.debug("SharePoint: Access token lookup failed for library '"+siteLibPath+"' - deleting"); + Logging.connectors.debug("SharePoint: Field list lookup failed for library '"+siteLibPath+"' - deleting"); activities.noDocument(documentIdentifier,version); } } else { if (Logging.connectors.isDebugEnabled()) - Logging.connectors.debug("SharePoint: Field list lookup failed for library '"+siteLibPath+"' - deleting"); + Logging.connectors.debug("SharePoint: GUID lookup failed for library '"+siteLibPath+"' - deleting"); activities.noDocument(documentIdentifier,version); } } - else - { - if (Logging.connectors.isDebugEnabled()) - Logging.connectors.debug("SharePoint: GUID lookup failed for library '"+siteLibPath+"' - deleting"); - activities.noDocument(documentIdentifier,version); - } } else {