Return-Path: X-Original-To: apmail-manifoldcf-commits-archive@www.apache.org Delivered-To: apmail-manifoldcf-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id C54EE10C63 for ; Sat, 23 Nov 2013 13:49:26 +0000 (UTC) Received: (qmail 45709 invoked by uid 500); 23 Nov 2013 13:49:24 -0000 Delivered-To: apmail-manifoldcf-commits-archive@manifoldcf.apache.org Received: (qmail 45682 invoked by uid 500); 23 Nov 2013 13:49:24 -0000 Mailing-List: contact commits-help@manifoldcf.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@manifoldcf.apache.org Delivered-To: mailing list commits@manifoldcf.apache.org Received: (qmail 45675 invoked by uid 99); 23 Nov 2013 13:49:23 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 23 Nov 2013 13:49:23 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 23 Nov 2013 13:49:19 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 453E82388999; Sat, 23 Nov 2013 13:48:58 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1544795 - in /manifoldcf/trunk: ./ connectors/sharepoint/ connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/sharepoint/ Date: Sat, 23 Nov 2013 13:48:58 -0000 To: commits@manifoldcf.apache.org From: kwright@apache.org X-Mailer: svnmailer-1.0.9 Message-Id: <20131123134858.453E82388999@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: kwright Date: Sat Nov 23 13:48:57 2013 New Revision: 1544795 URL: http://svn.apache.org/r1544795 Log: Fix for CONNECTORS-813 Modified: manifoldcf/trunk/ (props changed) manifoldcf/trunk/connectors/sharepoint/ (props changed) manifoldcf/trunk/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/sharepoint/IFileStream.java manifoldcf/trunk/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/sharepoint/SPSProxyHelper.java manifoldcf/trunk/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/sharepoint/SharePointRepository.java Propchange: manifoldcf/trunk/ ------------------------------------------------------------------------------ Merged /manifoldcf/branches/CONNECTORS-813:r1544010-1544792 Propchange: manifoldcf/trunk/connectors/sharepoint/ ------------------------------------------------------------------------------ Merged /manifoldcf/branches/CONNECTORS-813/connectors/sharepoint:r1544010-1544792 Modified: manifoldcf/trunk/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/sharepoint/IFileStream.java URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/sharepoint/IFileStream.java?rev=1544795&r1=1544794&r2=1544795&view=diff ============================================================================== --- manifoldcf/trunk/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/sharepoint/IFileStream.java (original) +++ manifoldcf/trunk/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/sharepoint/IFileStream.java Sat Nov 23 13:48:57 2013 @@ -20,6 +20,6 @@ import org.apache.manifoldcf.core.interf public interface IFileStream { - public void addFile(String relPath) + public void addFile(String relPath, String displayURI) throws ManifoldCFException; } Modified: manifoldcf/trunk/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/sharepoint/SPSProxyHelper.java URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/sharepoint/SPSProxyHelper.java?rev=1544795&r1=1544794&r2=1544795&view=diff ============================================================================== --- manifoldcf/trunk/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/sharepoint/SPSProxyHelper.java (original) +++ manifoldcf/trunk/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/sharepoint/SPSProxyHelper.java Sat Nov 23 13:48:57 2013 @@ -589,7 +589,7 @@ public class SPSProxyHelper { Object node = nodeDocs.get(j); Logging.connectors.debug( node.toString() ); String relPath = docs.getData( docs.getElement( node, "FileRef" ) ); - fileStream.addFile( relPath ); + fileStream.addFile( relPath, null ); } } else @@ -634,7 +634,8 @@ public class SPSProxyHelper { { resultCount++; String relPath = result.getAttribute("FileRef"); - fileStream.addFile( relPath ); + String displayURL = result.getAttribute("ListItemURL"); + fileStream.addFile( relPath, displayURL ); } } Modified: manifoldcf/trunk/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/sharepoint/SharePointRepository.java URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/sharepoint/SharePointRepository.java?rev=1544795&r1=1544794&r2=1544795&view=diff ============================================================================== --- manifoldcf/trunk/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/sharepoint/SharePointRepository.java (original) +++ manifoldcf/trunk/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/sharepoint/SharePointRepository.java Sat Nov 23 13:48:57 2013 @@ -782,13 +782,20 @@ public class SharePointRepository extend String[] denyTokens = activities.retrieveParentData(documentIdentifier, "denyTokens"); String[] listIDs = activities.retrieveParentData(documentIdentifier, "guids"); String[] listFields = activities.retrieveParentData(documentIdentifier, "fields"); - + String[] displayURLs = activities.retrieveParentData(documentIdentifier, "displayURLs"); + String listID; if (listIDs.length >= 1) listID = listIDs[0]; else listID = null; + String displayURL; + if (displayURLs.length >= 1) + displayURL = displayURLs[0]; + else + displayURL = null; + if (listID != null) { String[] sortedMetadataFields = getInterestingFieldSetSorted(metadataInfo,listFields); @@ -803,7 +810,6 @@ public class SharePointRepository extend metadataDescription.add("Created"); metadataDescription.add("ID"); metadataDescription.add("GUID"); - metadataDescription.add("EncodedAbsUrl"); // The document path includes the library, with no leading slash, and is decoded. String decodedItemPathWithoutSite = decodedItemPath.substring(cutoff+1); Map values = proxy.getFieldValues( metadataDescription, encodedSitePath, listID, "/Lists/" + decodedItemPathWithoutSite, dspStsWorks ); @@ -811,7 +817,6 @@ public class SharePointRepository extend String createdDate = values.get("Created"); String id = values.get("ID"); String guid = values.get("GUID"); - String absURL = values.get("EncodedAbsUrl"); if (modifiedDate != null) { // Item has a modified date so we presume it exists. @@ -833,7 +838,7 @@ public class SharePointRepository extend packDate(sb,createdDateValue); pack(sb,id,'+'); pack(sb,guid,'+'); - pack(sb,absURL,'+'); + pack(sb,displayURL,'+'); // The rest of this is unparseable sb.append(versionToken); sb.append(pathNameAttributeVersion); @@ -1374,9 +1379,9 @@ public class SharePointRepository extend String guid = guidBuffer.toString(); // List item URL - StringBuilder absURLBuffer = new StringBuilder(); - startPosition = unpack(absURLBuffer,version,startPosition,'+'); - String absURL = absURLBuffer.toString(); + StringBuilder relURLBuffer = new StringBuilder(); + startPosition = unpack(relURLBuffer,version,startPosition,'+'); + String relURL = relURLBuffer.toString(); // We need the list ID, which we've already fetched, so grab that from the parent data. String[] listIDs = activities.retrieveParentData(documentIdentifier, "guids"); @@ -1441,9 +1446,9 @@ public class SharePointRepository extend // Convert the modified document path to an unmodified one, plus a library path. String encodedItemPath = encodePath(decodedListPath.substring(0,listCutoff) + "/Lists/" + decodedItemPath.substring(listCutoff+1)); - // Generate the URL we are going to use - String itemUrl = fileBaseUrl + encodedItemPath; + String itemUrl = fileBaseUrl + relURL; //fileBaseUrl + encodedItemPath; + if (Logging.connectors.isDebugEnabled()) Logging.connectors.debug( "SharePoint: Processing list item '"+documentIdentifier+"'; url: '" + itemUrl + "'" ); @@ -1492,7 +1497,7 @@ public class SharePointRepository extend } data.addField("GUID",guid); - activities.ingestDocument( documentIdentifier, version, absURL , data ); + activities.ingestDocument( documentIdentifier, version, itemUrl , data ); } finally { @@ -2179,7 +2184,8 @@ public class SharePointRepository extend this.dataValues[3] = fields; } - public void addFile(String relPath) + @Override + public void addFile(String relPath, String displayURL) throws ManifoldCFException { @@ -2217,7 +2223,7 @@ public class SharePointRepository extend } } - protected final static String[] listItemStreamDataNames = new String[]{"accessTokens", "denyTokens", "guids", "fields"}; + protected final static String[] listItemStreamDataNames = new String[]{"accessTokens", "denyTokens", "guids", "fields", "displayURLs"}; protected class ListItemStream implements IFileStream { @@ -2247,7 +2253,8 @@ public class SharePointRepository extend this.dataValues[3] = fields; } - public void addFile(String relPath) + @Override + public void addFile(String relPath, String displayURL) throws ManifoldCFException { // First, convert the relative path to a full path @@ -2256,6 +2263,8 @@ public class SharePointRepository extend relPath = rootPath + sitePath + "/" + relPath; } + String fullPath = relPath; + // Now, strip away what we don't want - namely, the root path. This makes the path relative to the root. if ( relPath.startsWith(rootPath) ) { @@ -2277,6 +2286,11 @@ public class SharePointRepository extend // The way I've chosen to do this is to use a triple slash at that point, as a separator. String modifiedPath = relPath.substring(0,siteListPath.length()) + "//" + relPath.substring(siteListPath.length()); + if (displayURL != null) + dataValues[4] = new String[]{displayURL}; + else + dataValues[4] = new String[]{fullPath}; + activities.addDocumentReference( modifiedPath, documentIdentifier, null, listItemStreamDataNames, dataValues ); } else