manifoldcf-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kwri...@apache.org
Subject svn commit: r1381232 - in /manifoldcf/trunk: ./ CHANGES.txt connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/sharepoint/SPSProxyHelper.java connectors/sharepoint/wsdls/MCPermissions.wsdl
Date Wed, 05 Sep 2012 16:15:34 GMT
Author: kwright
Date: Wed Sep  5 16:15:33 2012
New Revision: 1381232

URL: http://svn.apache.org/viewvc?rev=1381232&view=rev
Log:
Merge in changes for CONNECTORS-492.

Modified:
    manifoldcf/trunk/   (props changed)
    manifoldcf/trunk/CHANGES.txt
    manifoldcf/trunk/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/sharepoint/SPSProxyHelper.java
    manifoldcf/trunk/connectors/sharepoint/wsdls/MCPermissions.wsdl

Propchange: manifoldcf/trunk/
------------------------------------------------------------------------------
  Merged /manifoldcf/branches/CONNECTORS-492:r1368611-1381230

Modified: manifoldcf/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/CHANGES.txt?rev=1381232&r1=1381231&r2=1381232&view=diff
==============================================================================
--- manifoldcf/trunk/CHANGES.txt (original)
+++ manifoldcf/trunk/CHANGES.txt Wed Sep  5 16:15:33 2012
@@ -5,6 +5,10 @@ $Id$
 CONNECTORS-521: Fix project site's broken search
 (Ahmet Arslan)
 
+CONNECTORS-492: Replace Lists.GetListItems call with a custom call,
+in SharePoint connector, to dodge limits on size of response.
+(Ahmet Arslan, Piergiorgio Lucidi, Joe Becknell, Karl Wright)
+
 CONNECTORS-522: Wiki connector namespace tab was non-functional.
 (Maciej Lizewski, Karl Wright)
 

Modified: manifoldcf/trunk/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/sharepoint/SPSProxyHelper.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/sharepoint/SPSProxyHelper.java?rev=1381232&r1=1381231&r2=1381232&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/sharepoint/SPSProxyHelper.java
(original)
+++ manifoldcf/trunk/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/sharepoint/SPSProxyHelper.java
Wed Sep  5 16:15:33 2012
@@ -296,7 +296,7 @@ public class SPSProxyHelper {
 
   /**
   * Get the acls for a document.
-  * NOTE that this function only works for SharePoint 3.0 with the MCPermissions web service
installed.
+  * NOTE that this function only works for SharePoint 2007+ with the MCPermissions web service
installed.
   * @param site is the encoded subsite path
   * @param file is the encoded file url (not including protocol or server or location, but
including encoded subsite, library and folder/file path)
   * @return array of document SIDs
@@ -607,98 +607,69 @@ public class SPSProxyHelper {
       }
       else
       {
-        // Sharepoint 2010; use Lists service instead
-        ListsWS lservice = new ListsWS(baseUrl + site, userName, password, myFactory, configuration,
connectionManager );
-        ListsSoapStub stub1 = (ListsSoapStub)lservice.getListsSoapHandler();
+        // New code
+        
+        MCPermissionsWS itemService = new MCPermissionsWS( baseUrl + site, userName, password,
myFactory, configuration, connectionManager );
+        com.microsoft.sharepoint.webpartpages.PermissionsSoap itemCall = itemService.getPermissionsSoapHandler(
);
 
-        // This string is the paging chunk description.  It gets updated on every chunk we
do,
-        // so that the next call finds new data.
-        String nextChunkDescription = "";
-        // Order by some column we know is indexed.
-        GetListItemsQuery orderByQuery = buildOrderedQuery("ID");
-        // Set up fields we want
-        ArrayList fieldList = new ArrayList();
-        fieldList.add("FileRef");
-        GetListItemsViewFields viewFields = buildViewFields(fieldList);
-        // Pick a request size we know will not exceed the limit as set by the administrator.
-        int requestSize = 2000;
-          
+        int startingIndex = 0;
+        int amtToRequest = 1000;
         while (true)
         {
-          GetListItemsResponseGetListItemsResult items =  stub1.getListItems(guid, "", orderByQuery,
viewFields, Integer.toString(requestSize), buildPagingQueryOptions(nextChunkDescription),
null);
-          if (items == null)
-            return false;
-
-          org.apache.axis.message.MessageElement[] list = items.get_any();
+          com.microsoft.sharepoint.webpartpages.GetListItemsResponseGetListItemsResult itemsResult
=
+            itemCall.getListItems(guid,Integer.toString(startingIndex),Integer.toString(amtToRequest));
+          
+          MessageElement[] itemsList = itemsResult.get_any();
 
           if (Logging.connectors.isDebugEnabled()){
-            Logging.connectors.debug("SharePoint: getListItems xml response: '" + list[0].toString()
+ "'");
+            Logging.connectors.debug("SharePoint: getListItems xml response: '" + itemsList[0].toString()
+ "'");
           }
 
-          ArrayList nodeList = new ArrayList();
-          XMLDoc doc = new XMLDoc(list[0].toString());
-
-          doc.processPath(nodeList, "*", null);
-          if (nodeList.size() != 1) {
-            throw new ManifoldCFException("Bad xml - expecting one outer 'ns1:listitems'
node - there are " + Integer.toString(nodeList.size()) + " nodes");
-          }
-
-          Object parent = nodeList.get(0);
-          if (!"ns1:listitems".equals(doc.getNodeName(parent)))
-            throw new ManifoldCFException("Bad xml - outer node is not 'ns1:listitems'");
-
-
-          nodeList.clear();
-          doc.processPath(nodeList, "*", parent);
-
-          if (nodeList.size() != 1)
-            throw new ManifoldCFException("Expected rsdata result but no results found.");
-
-          Object rsData = nodeList.get(0);
-
-          // Get the chunk description
-          nextChunkDescription = doc.getValue(rsData, "ListItemCollectionPositionNext");
-
-          int itemCount = Integer.parseInt(doc.getValue(rsData, "ItemCount"));
-
-          // Now, extract the files from the response document
-          XMLDoc docs = doc;
-          ArrayList nodeDocs = new ArrayList();
-
-          docs.processPath(nodeDocs, "*", rsData);
-
-          if (nodeDocs.size() != itemCount) {
-            throw new ManifoldCFException("itemCount does not match with nodeDocs.size().");
-          }
-
-          for (int j = 0; j < nodeDocs.size(); j++)
+          if (itemsList.length != 1)
+            throw new ManifoldCFException("Bad response - expecting one outer 'GetListItems'
node, saw "+Integer.toString(itemsList.length));
+          
+          MessageElement items = itemsList[0];
+          if (!items.getElementName().getLocalName().equals("GetListItems"))
+            throw new ManifoldCFException("Bad response - outer node should have been 'GetListItems'
node");
+          
+          int resultCount = 0;
+          Iterator iter = items.getChildElements();
+          while (iter.hasNext())
           {
-
-            Object node = nodeDocs.get(j);
-
-            String relPath = doc.getValue(node, "ows_FileRef");
-
-            // This relative path is apparently from the domain on down; if there's a location
offset we therefore
-            // need to get rid of it before checking the document against the site/library
tuples.  The recorded
-            // document identifier should also not include it.
-
-            // KDW: Removed the case changes; URL characters should remain case-sensitive
-            if (!relPath.startsWith(serverLocation))
+            MessageElement child = (MessageElement)iter.next();
+            if (child.getElementName().getLocalName().equals("GetListItemsResponse"))
             {
-              // Unexpected processing error; the path to the folder or document did not
start with the location
-              // offset, so throw up.
-              throw new ManifoldCFException("Internal error: Relative path '"+relPath+"'
was expected to start with '"+
-                serverLocation+"'");
+              Iterator resultIter = child.getChildElements();
+              while (resultIter.hasNext())
+              {
+                MessageElement result = (MessageElement)resultIter.next();
+                if (result.getElementName().getLocalName().equals("GetListItemsResult"))
+                {
+                  resultCount++;
+                  String relPath = result.getAttribute("FileRef");
+                  // This includes the document library, so munge the path accordingly
+                  if (!relPath.startsWith(serverLocation))
+                  {
+                    // Unexpected processing error; the path to the folder or document did
not start with the location
+                    // offset, so throw up.
+                    throw new ManifoldCFException("Internal error: Relative path '"+relPath+"'
was expected to start with '"+
+                      serverLocation+"'");
+                  }
+                  relPath = relPath.substring(serverLocation.length());
+
+                  relPath = "/" + relPath;
+
+                  fileStream.addFile( relPath );
+                }
+              }
+              
             }
-            relPath = relPath.substring(serverLocation.length());
-
-            relPath = "/" + valueMunge(relPath);
-
-            fileStream.addFile( relPath );
           }
           
-          if (requestSize > nodeDocs.size())
+          if (resultCount < amtToRequest)
             break;
+          
+          startingIndex += resultCount;
         }
       }
       

Modified: manifoldcf/trunk/connectors/sharepoint/wsdls/MCPermissions.wsdl
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/sharepoint/wsdls/MCPermissions.wsdl?rev=1381232&r1=1381231&r2=1381232&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/sharepoint/wsdls/MCPermissions.wsdl (original)
+++ manifoldcf/trunk/connectors/sharepoint/wsdls/MCPermissions.wsdl Wed Sep  5 16:15:33 2012
@@ -1,18 +1,4 @@
-<!-- Licensed to the Apache Software Foundation (ASF) under one or more
-     contributor license agreements. See the NOTICE file distributed with
-     this work for additional information regarding copyright ownership.
-     The ASF licenses this file to You under the Apache License, Version 2.0
-     (the "License"); you may not use this file except in compliance with
-     the License. You may obtain a copy of the License at
 
-     http://www.apache.org/licenses/LICENSE-2.0
-
-     Unless required by applicable law or agreed to in writing, software
-     distributed under the License is distributed on an "AS IS" BASIS,
-     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-     See the License for the specific language governing permissions and
-     limitations under the License.
--->
 <wsdl:definitions xmlns:soap="http://schemas.xmlsoap.org/wsdl/soap/" xmlns:tm="http://microsoft.com/wsdl/mime/textMatching/"
xmlns:soapenc="http://schemas.xmlsoap.org/soap/encoding/" xmlns:mime="http://schemas.xmlsoap.org/wsdl/mime/"
xmlns:tns="http://microsoft.com/sharepoint/webpartpages/" xmlns:s="http://www.w3.org/2001/XMLSchema"
xmlns:soap12="http://schemas.xmlsoap.org/wsdl/soap12/" xmlns:http="http://schemas.xmlsoap.org/wsdl/http/"
targetNamespace="http://microsoft.com/sharepoint/webpartpages/" xmlns:wsdl="http://schemas.xmlsoap.org/wsdl/">
   <wsdl:types>
     <s:schema elementFormDefault="qualified" targetNamespace="http://microsoft.com/sharepoint/webpartpages/">
@@ -22,7 +8,6 @@
             <s:element minOccurs="0" maxOccurs="1" name="objectName" type="s:string" />
             <s:element minOccurs="0" maxOccurs="1" name="objectType" type="s:string" />
           </s:sequence>
-
         </s:complexType>
       </s:element>
       <s:element name="GetPermissionCollectionResponse">
@@ -32,7 +17,28 @@
               <s:complexType mixed="true">
                 <s:sequence>
                   <s:any />
-
+                </s:sequence>
+              </s:complexType>
+            </s:element>
+          </s:sequence>
+        </s:complexType>
+      </s:element>
+      <s:element name="GetListItems">
+        <s:complexType>
+          <s:sequence>
+            <s:element minOccurs="0" maxOccurs="1" name="listName" type="s:string" />
+            <s:element minOccurs="0" maxOccurs="1" name="startRow" type="s:string" />
+            <s:element minOccurs="0" maxOccurs="1" name="rowLimit" type="s:string" />
+          </s:sequence>
+        </s:complexType>
+      </s:element>
+      <s:element name="GetListItemsResponse">
+        <s:complexType>
+          <s:sequence>
+            <s:element minOccurs="0" maxOccurs="1" name="GetListItemsResult">
+              <s:complexType mixed="true">
+                <s:sequence>
+                  <s:any />
                 </s:sequence>
               </s:complexType>
             </s:element>
@@ -42,26 +48,43 @@
     </s:schema>
   </wsdl:types>
   <wsdl:message name="GetPermissionCollectionSoapIn">
-
     <wsdl:part name="parameters" element="tns:GetPermissionCollection" />
   </wsdl:message>
   <wsdl:message name="GetPermissionCollectionSoapOut">
     <wsdl:part name="parameters" element="tns:GetPermissionCollectionResponse" />
   </wsdl:message>
+  <wsdl:message name="GetListItemsSoapIn">
+    <wsdl:part name="parameters" element="tns:GetListItems" />
+  </wsdl:message>
+  <wsdl:message name="GetListItemsSoapOut">
+    <wsdl:part name="parameters" element="tns:GetListItemsResponse" />
+  </wsdl:message>
   <wsdl:portType name="PermissionsSoap">
     <wsdl:operation name="GetPermissionCollection">
       <wsdl:documentation xmlns:wsdl="http://schemas.xmlsoap.org/wsdl/">Returns the
collection of permissions for a site, list, or list item.</wsdl:documentation>
-
       <wsdl:input message="tns:GetPermissionCollectionSoapIn" />
       <wsdl:output message="tns:GetPermissionCollectionSoapOut" />
     </wsdl:operation>
+    <wsdl:operation name="GetListItems">
+      <wsdl:documentation xmlns:wsdl="http://schemas.xmlsoap.org/wsdl/">Returns the
list of contents of a library without interference from administrator-set limits.</wsdl:documentation>
+      <wsdl:input message="tns:GetListItemsSoapIn" />
+      <wsdl:output message="tns:GetListItemsSoapOut" />
+    </wsdl:operation>
   </wsdl:portType>
   <wsdl:binding name="PermissionsSoap" type="tns:PermissionsSoap">
     <soap:binding transport="http://schemas.xmlsoap.org/soap/http" />
     <wsdl:operation name="GetPermissionCollection">
       <soap:operation soapAction="http://microsoft.com/sharepoint/webpartpages/GetPermissionCollection"
style="document" />
       <wsdl:input>
-
+        <soap:body use="literal" />
+      </wsdl:input>
+      <wsdl:output>
+        <soap:body use="literal" />
+      </wsdl:output>
+    </wsdl:operation>
+    <wsdl:operation name="GetListItems">
+      <soap:operation soapAction="http://microsoft.com/sharepoint/webpartpages/GetListItems"
style="document" />
+      <wsdl:input>
         <soap:body use="literal" />
       </wsdl:input>
       <wsdl:output>
@@ -71,7 +94,6 @@
   </wsdl:binding>
   <wsdl:binding name="PermissionsSoap12" type="tns:PermissionsSoap">
     <soap12:binding transport="http://schemas.xmlsoap.org/soap/http" />
-
     <wsdl:operation name="GetPermissionCollection">
       <soap12:operation soapAction="http://microsoft.com/sharepoint/webpartpages/GetPermissionCollection"
style="document" />
       <wsdl:input>
@@ -81,15 +103,22 @@
         <soap12:body use="literal" />
       </wsdl:output>
     </wsdl:operation>
-
+    <wsdl:operation name="GetListItems">
+      <soap12:operation soapAction="http://microsoft.com/sharepoint/webpartpages/GetListItems"
style="document" />
+      <wsdl:input>
+        <soap12:body use="literal" />
+      </wsdl:input>
+      <wsdl:output>
+        <soap12:body use="literal" />
+      </wsdl:output>
+    </wsdl:operation>
   </wsdl:binding>
   <wsdl:service name="Permissions">
     <wsdl:port name="PermissionsSoap" binding="tns:PermissionsSoap">
-      <soap:address location="http://w2k3-shp-76-2.qa-ad-76.metacarta.com/_vti_bin/MCPermissions.asmx"
/>
+      <soap:address location="http://win-0hb0c25kl3n:33445/_vti_bin/MCPermissions.asmx"
/>
     </wsdl:port>
     <wsdl:port name="PermissionsSoap12" binding="tns:PermissionsSoap12">
-      <soap12:address location="http://w2k3-shp-76-2.qa-ad-76.metacarta.com/_vti_bin/MCPermissions.asmx"
/>
+      <soap12:address location="http://win-0hb0c25kl3n:33445/_vti_bin/MCPermissions.asmx"
/>
     </wsdl:port>
   </wsdl:service>
-
-</wsdl:definitions>
+</wsdl:definitions>
\ No newline at end of file



Mime
View raw message