manifoldcf-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kwri...@apache.org
Subject svn commit: r1629377 - in /manifoldcf/branches/CONNECTORS-1037: connectors/alfresco-webscript/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/alfrescowebscript/ connectors/alfresco-webscript/connector/src/main/java/org/apache/mani...
Date Sat, 04 Oct 2014 12:12:44 GMT
Author: kwright
Date: Sat Oct  4 12:12:43 2014
New Revision: 1629377

URL: http://svn.apache.org/r1629377
Log:
Fix tests, url generation, document filtering, and obfuscation of password values

Modified:
    manifoldcf/branches/CONNECTORS-1037/connectors/alfresco-webscript/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/alfrescowebscript/AlfrescoAuthorityConnector.java
    manifoldcf/branches/CONNECTORS-1037/connectors/alfresco-webscript/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/alfrescowebscript/ConfigurationHandler.java
    manifoldcf/branches/CONNECTORS-1037/connectors/alfresco-webscript/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/alfrescowebscript/AlfrescoConnector.java
    manifoldcf/branches/CONNECTORS-1037/connectors/alfresco-webscript/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/alfrescowebscript/ConfigurationHandler.java
    manifoldcf/branches/CONNECTORS-1037/connectors/alfresco-webscript/connector/src/test/java/org/apache/manifoldcf/crawler/connectors/alfrescowebscript/AlfrescoConnectorTest.java
    manifoldcf/branches/CONNECTORS-1037/framework/core/src/main/java/org/apache/manifoldcf/core/common/DateParser.java
    manifoldcf/branches/CONNECTORS-1037/framework/core/src/test/java/org/apache/manifoldcf/core/common/DateTest.java

Modified: manifoldcf/branches/CONNECTORS-1037/connectors/alfresco-webscript/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/alfrescowebscript/AlfrescoAuthorityConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1037/connectors/alfresco-webscript/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/alfrescowebscript/AlfrescoAuthorityConnector.java?rev=1629377&r1=1629376&r2=1629377&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-1037/connectors/alfresco-webscript/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/alfrescowebscript/AlfrescoAuthorityConnector.java
(original)
+++ manifoldcf/branches/CONNECTORS-1037/connectors/alfresco-webscript/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/alfrescowebscript/AlfrescoAuthorityConnector.java
Sat Oct  4 12:12:43 2014
@@ -48,7 +48,7 @@ public class AlfrescoAuthorityConnector 
     String hostname = getConfig(config, "hostname", "localhost");
     String endpoint = getConfig(config, "endpoint", "/alfresco/service");
     String username = getConfig(config, "username", null);
-    String password = getConfig(config, "password", null);
+    String password = getObfuscatedConfig(config, "password", null);
 
     alfrescoClient = new WebScriptsAlfrescoClient(protocol, hostname, endpoint,
         null, null, username, password);
@@ -64,6 +64,16 @@ public class AlfrescoAuthorityConnector 
     return protocol;
   }
 
+  private static String getObfuscatedConfig(ConfigParams config,
+                                  String parameter,
+                                  String defaultValue) {
+    final String protocol = config.getObfuscatedParameter(parameter);
+    if (protocol == null) {
+      return defaultValue;
+    }
+    return protocol;
+  }
+
   /*
    * (non-Javadoc)
    * @see org.apache.manifoldcf.core.connector.BaseConnector#check()

Modified: manifoldcf/branches/CONNECTORS-1037/connectors/alfresco-webscript/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/alfrescowebscript/ConfigurationHandler.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1037/connectors/alfresco-webscript/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/alfrescowebscript/ConfigurationHandler.java?rev=1629377&r1=1629376&r2=1629377&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-1037/connectors/alfresco-webscript/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/alfrescowebscript/ConfigurationHandler.java
(original)
+++ manifoldcf/branches/CONNECTORS-1037/connectors/alfresco-webscript/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/alfrescowebscript/ConfigurationHandler.java
Sat Oct  4 12:12:43 2014
@@ -59,19 +59,28 @@ public class ConfigurationHandler {
                                                List<String> tabsArray) throws ManifoldCFException,
IOException {
     tabsArray.add("Server");
     Map<String, Object> paramMap = new HashMap<String, Object>();
-    fillInParameters(paramMap, parameters);
+    fillInParameters(paramMap, out, parameters);
     Messages.outputResourceWithVelocity(out, locale, EDIT_CONFIG_HEADER, paramMap);
   }
 
   private static void fillInParameters(Map<String, Object> paramMap,
-                                       ConfigParams parameters) {
+                                       IHTTPOutput out, ConfigParams parameters) {
     for (Map.Entry<String, String> parameter : DEFAULT_CONFIGURATION_PARAMETERS
         .entrySet()) {
-      String paramValue = parameters.getParameter(parameter.getKey());
-      if (paramValue == null) {
-        paramValue = parameter.getValue();
+      String paramName = parameter.getKey();
+      if (paramName.endsWith("password")) {
+        String paramValue = parameters.getObfuscatedParameter(paramName);
+        if (paramValue == null) {
+          paramValue = parameter.getValue();
+        }
+        paramMap.put(paramName, out.mapPasswordToKey(paramValue));
+      } else {
+        String paramValue = parameters.getParameter(paramName);
+        if (paramValue == null) {
+          paramValue = parameter.getValue();
+        }
+        paramMap.put(paramName, paramValue);
       }
-      paramMap.put(parameter.getKey(), paramValue);
     }
   }
 
@@ -81,7 +90,7 @@ public class ConfigurationHandler {
       throws ManifoldCFException, IOException {
     Map<String, Object> paramMap = new HashMap<String, Object>();
     paramMap.put("tabName", tabName);
-    fillInParameters(paramMap, parameters);
+    fillInParameters(paramMap, out, parameters);
     Messages.outputResourceWithVelocity(out, locale, EDIT_CONFIG_SERVER, paramMap);
   }
 
@@ -92,7 +101,10 @@ public class ConfigurationHandler {
     for (String paramName : DEFAULT_CONFIGURATION_PARAMETERS.keySet()) {
       String paramValue = variableContext.getParameter(paramName);
       if (paramValue != null) {
-        parameters.setParameter(paramName, paramValue);
+        if (paramName.endsWith("password"))
+          parameters.setObfuscatedParameter(paramName,variableContext.mapKeyToPassword(paramValue));
+        else
+          parameters.setParameter(paramName, paramValue);
       }
     }
     return null;
@@ -103,7 +115,7 @@ public class ConfigurationHandler {
                                        ConfigParams parameters)
       throws ManifoldCFException, IOException {
     Map<String, Object> paramMap = new HashMap<String, Object>();
-    fillInParameters(paramMap, parameters);
+    fillInParameters(paramMap, out, parameters);
     Messages.outputResourceWithVelocity(out, locale, VIEW_CONFIG, paramMap);
   }
 }

Modified: manifoldcf/branches/CONNECTORS-1037/connectors/alfresco-webscript/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/alfrescowebscript/AlfrescoConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1037/connectors/alfresco-webscript/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/alfrescowebscript/AlfrescoConnector.java?rev=1629377&r1=1629376&r2=1629377&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-1037/connectors/alfresco-webscript/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/alfrescowebscript/AlfrescoConnector.java
(original)
+++ manifoldcf/branches/CONNECTORS-1037/connectors/alfresco-webscript/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/alfrescowebscript/AlfrescoConnector.java
Sat Oct  4 12:12:43 2014
@@ -23,6 +23,7 @@ import org.alfresco.consulting.indexer.c
 import org.apache.manifoldcf.agents.interfaces.RepositoryDocument;
 import org.apache.manifoldcf.agents.interfaces.ServiceInterruption;
 import org.apache.manifoldcf.core.interfaces.*;
+import org.apache.manifoldcf.core.common.DateParser;
 import org.apache.manifoldcf.crawler.connectors.BaseRepositoryConnector;
 import org.apache.manifoldcf.crawler.interfaces.IExistingVersions;
 import org.apache.manifoldcf.crawler.interfaces.IProcessActivity;
@@ -31,6 +32,7 @@ import org.apache.manifoldcf.crawler.sys
 
 import java.io.ByteArrayInputStream;
 import java.io.IOException;
+import java.io.InterruptedIOException;
 import java.io.InputStream;
 import java.text.MessageFormat;
 import java.text.ParseException;
@@ -76,7 +78,7 @@ public class AlfrescoConnector extends B
     String storeProtocol = getConfig(config, "storeprotocol", "workspace");
     String storeId = getConfig(config, "storeid", "SpacesStore");
     String username = getConfig(config, "username", null);
-    String password = getConfig(config, "password", null);
+    String password = getObfuscatedConfig(config, "password", null);
     this.enableDocumentProcessing = new Boolean(getConfig(config, "enabledocumentprocessing",
"false"));
 
     alfrescoClient = new WebScriptsAlfrescoClient(protocol, hostname, endpoint,
@@ -93,9 +95,28 @@ public class AlfrescoConnector extends B
     return protocol;
   }
 
+  private static String getObfuscatedConfig(ConfigParams config,
+                                  String parameter,
+                                  String defaultValue) {
+    final String protocol = config.getObfuscatedParameter(parameter);
+    if (protocol == null) {
+      return defaultValue;
+    }
+    return protocol;
+  }
+
   @Override
   public String check() throws ManifoldCFException {
-    return super.check();
+    try {
+      // We really want to do something more like fetching a document here...
+      alfrescoClient.fetchUserAuthorities("admin");
+      return super.check();
+    } catch (AlfrescoDownException e) {
+      if (Logging.connectors != null) {
+        Logging.connectors.warn(e.getMessage(), e);
+      }
+      return "Connection failed: " + e.getMessage();
+    }
   }
 
   @Override
@@ -164,7 +185,8 @@ public class AlfrescoConnector extends B
         Logging.connectors.debug(MessageFormat.format("Recording {0} as last transaction
id and {1} as last changeset id", new Object[]{lastTransactionId, lastAclChangesetId}));
       return lastTransactionId + "|" + lastAclChangesetId;
     } catch (AlfrescoDownException e) {
-      throw new ManifoldCFException(e);
+      handleAlfrescoDownException(e,"seeding");
+      return null;
     }
   }
 
@@ -173,121 +195,128 @@ public class AlfrescoConnector extends B
                                IProcessActivity activities, int jobMode, boolean usesDefaultAuthority)
     throws ManifoldCFException, ServiceInterruption {
 
-    for (String doc : documentIdentifiers) {
+    try {
+      for (String doc : documentIdentifiers) {
 
-      String nextVersion = statuses.getIndexedVersionString(doc);	
-    	
-      // Calling again Alfresco API because Document's actions are lost from seeding method
-      AlfrescoResponse response = alfrescoClient.fetchNode(doc);
-      if(response.getDocumentList().isEmpty()){ // Not found seeded document. Could reflect
an error in Alfresco
-        if (Logging.connectors != null)
-          Logging.connectors.warn(MessageFormat.format("Invalid Seeded Document from Alfresco
with ID {0}", new Object[]{doc}));
-        activities.deleteDocument(doc);
-        continue;
-      }
-      Map<String, Object> map = response.getDocumentList().get(0); // Should be only
one
-      if ((Boolean) map.get("deleted")) {
-        activities.deleteDocument(doc);
-        continue;
-      }
+        String nextVersion = statuses.getIndexedVersionString(doc);	
+          
+        // Calling again Alfresco API because Document's actions are lost from seeding method
+        AlfrescoResponse response = alfrescoClient.fetchNode(doc);
+        if(response.getDocumentList().isEmpty()){ // Not found seeded document. Could reflect
an error in Alfresco
+          if (Logging.connectors != null)
+            Logging.connectors.warn(MessageFormat.format("Invalid Seeded Document from Alfresco
with ID {0}", new Object[]{doc}));
+          activities.deleteDocument(doc);
+          continue;
+        }
+        Map<String, Object> map = response.getDocumentList().get(0); // Should be only
one
+        if ((Boolean) map.get("deleted")) {
+          activities.deleteDocument(doc);
+          continue;
+        }
 
-      // From the map, get the things we know about
-      String uuid = map.containsKey(FIELD_UUID) ? map.get(FIELD_UUID).toString() : doc;
-      String nodeRef = map.containsKey(FIELD_NODEREF) ? map.get(FIELD_NODEREF).toString()
: "";
-      String type = map.containsKey(FIELD_TYPE) ? map.get(FIELD_TYPE).toString() : "";
-      String name =  map.containsKey(FIELD_NAME) ? map.get(FIELD_NAME).toString() : "";
-
-      // Fetch document metadata
-      Map<String,Object> properties = alfrescoClient.fetchMetadata(uuid);
-
-      // Process various special fields
-      Object mdObject;
-
-      // Size
-      Long lSize = null;
-      mdObject = properties.get(SIZE_PROPERTY);
-      if (mdObject != null) {
-        String size = mdObject.toString();
-        lSize = new Long(size);
-      }
-      
-      // Modified Date
-      Date modifiedDate = null;
-      mdObject = properties.get(MODIFIED_DATE_PROPERTY);
-      if (mdObject != null) {
-        SimpleDateFormat f = new SimpleDateFormat("YYYY-MM-DDThh:mm:ss.sTZD");
-        try {
-          modifiedDate = f.parse(mdObject.toString());
-        } catch (ParseException e) {
-          // Nothing to do
+        // From the map, get the things we know about
+        String uuid = doc;
+        String nodeRef = map.containsKey(FIELD_NODEREF) ? map.get(FIELD_NODEREF).toString()
: "";
+        String type = map.containsKey(FIELD_TYPE) ? map.get(FIELD_TYPE).toString() : "";
+        String name =  map.containsKey(FIELD_NAME) ? map.get(FIELD_NAME).toString() : "";
+
+        // Fetch document metadata
+        Map<String,Object> properties = alfrescoClient.fetchMetadata(uuid);
+
+        // Process various special fields
+        Object mdObject;
+
+        // Size
+        Long lSize = null;
+        mdObject = properties.get(SIZE_PROPERTY);
+        if (mdObject != null) {
+          String size = mdObject.toString();
+          lSize = new Long(size);
+        }
+        
+        // Modified Date
+        Date modifiedDate = null;
+        mdObject = properties.get(MODIFIED_DATE_PROPERTY);
+        if (mdObject != null) {
+          modifiedDate = DateParser.parseISO8601Date(mdObject.toString());
         }
-      }
 
-      // Created Date
-      Date createdDate = null;
-      mdObject = properties.get(CREATED_DATE_PROPERTY);
-      if (mdObject != null) {
-        SimpleDateFormat f = new SimpleDateFormat("YYYY-MM-DDThh:mm:ss.sTZD");
-        try {
-          createdDate = f.parse(mdObject.toString());
-        } catch (ParseException e) {
-          // Nothing to do
+        // Created Date
+        Date createdDate = null;
+        mdObject = properties.get(CREATED_DATE_PROPERTY);
+        if (mdObject != null) {
+          createdDate = DateParser.parseISO8601Date(mdObject.toString());
         }
-      }
 
 
-      // Establish the document version.
-      if (modifiedDate == null) {
-        activities.deleteDocument(doc);
-        continue;
-      }
-      
-      String documentVersion = modifiedDate.toString();
+        // Establish the document version.
+        if (modifiedDate == null) {
+          activities.deleteDocument(doc);
+          continue;
+        }
+        
+        String documentVersion = new Long(modifiedDate.getTime()).toString();
 
-      if(!activities.checkDocumentNeedsReindexing(doc, documentVersion))
-        continue;
+        if(!activities.checkDocumentNeedsReindexing(doc, documentVersion))
+          continue;
 
-      RepositoryDocument rd = new RepositoryDocument();
-      rd.addField(FIELD_NODEREF, nodeRef);
-      rd.addField(FIELD_TYPE, type);
-      rd.setFileName(name);
-      
-      if (modifiedDate != null)
-        rd.setModifiedDate(modifiedDate);
+        String mimeType = null;
+        Object mimetypeObject = properties.get(MIMETYPE_PROPERTY);
+        if (mimetypeObject != null) {
+          mimeType = mimetypeObject.toString();
+        }
 
-      if (createdDate != null)
-        rd.setCreatedDate(createdDate);
+        if (lSize != null && !activities.checkLengthIndexable(lSize.longValue()))
{
+          activities.noDocument(doc, documentVersion);
+          continue;
+        }
+        
+        if (mimeType != null && !activities.checkMimeTypeIndexable(mimeType)) {
+          activities.noDocument(doc, documentVersion);
+          continue;
+        }
 
-      for(String property : properties.keySet()) {
-        Object propertyValue = properties.get(property);
-        rd.addField(property,propertyValue.toString());
-      }
+        RepositoryDocument rd = new RepositoryDocument();
+        rd.addField(FIELD_NODEREF, nodeRef);
+        rd.addField(FIELD_TYPE, type);
+        rd.setFileName(name);
+        
+        if (modifiedDate != null)
+          rd.setModifiedDate(modifiedDate);
 
-      Object mimetypeObject = properties.get(MIMETYPE_PROPERTY);
-      if (mimetypeObject != null) {
-        String mimetype = mimetypeObject.toString();
-        if (mimetype != null && !mimetype.isEmpty())
-          rd.setMimeType(mimetype);
-      }
+        if (createdDate != null)
+          rd.setCreatedDate(createdDate);
 
-      // Indexing Permissions
-      @SuppressWarnings("unchecked")
-      List<String> permissions = (List<String>) properties.remove(AUTHORITIES_PROPERTY);
-      if(permissions != null){
-        rd.setSecurityACL(RepositoryDocument.SECURITY_TYPE_DOCUMENT,
-                          permissions.toArray(new String[permissions.size()]));
-      }
+        for(String property : properties.keySet()) {
+          Object propertyValue = properties.get(property);
+          rd.addField(property,propertyValue.toString());
+        }
+
+        if (mimeType != null && !mimeType.isEmpty())
+          rd.setMimeType(mimeType);
+
+        // Indexing Permissions
+        @SuppressWarnings("unchecked")
+        List<String> permissions = (List<String>) properties.remove(AUTHORITIES_PROPERTY);
+        if(permissions != null){
+          rd.setSecurityACL(RepositoryDocument.SECURITY_TYPE_DOCUMENT,
+                            permissions.toArray(new String[permissions.size()]));
+        }
 
-      // Document Binary Content
-      InputStream stream;
-      long length;
-      String contentUrlPath = (String) properties.get(CONTENT_URL_PROPERTY);
-      if (contentUrlPath != null && !contentUrlPath.isEmpty()) {
+        // Document Binary Content
+        InputStream stream;
+        long length;
         byte[] empty = new byte[0];
-        if (this.enableDocumentProcessing) {
-          if (lSize != null) {
-            stream = alfrescoClient.fetchContent(contentUrlPath);
-            length = lSize.longValue();
+        String contentUrlPath = (String) properties.get(CONTENT_URL_PROPERTY);
+        if (contentUrlPath != null && !contentUrlPath.isEmpty()) {
+          if (this.enableDocumentProcessing) {
+            if (lSize != null) {
+              stream = alfrescoClient.fetchContent(contentUrlPath);
+              length = lSize.longValue();
+            } else {
+              stream = new ByteArrayInputStream(empty);
+              length = 0L;
+            }
           } else {
             stream = new ByteArrayInputStream(empty);
             length = 0L;
@@ -301,27 +330,122 @@ public class AlfrescoConnector extends B
           rd.setBinary(stream, length);
           if (Logging.connectors != null && Logging.connectors.isDebugEnabled())
             Logging.connectors.debug(MessageFormat.format("Ingesting with id: {0}, URI {1}
and rd {2}", new Object[]{uuid, nodeRef, rd.getFileName()}));
-          activities.ingestDocumentWithException(doc, documentVersion, uuid, rd);
+          activities.ingestDocumentWithException(doc, documentVersion, nodeRef/*was uuid*/,
rd);
         } catch (IOException e) {
-          handleIOException(e);
+          handleIOException(e,"reading stream");
         } finally {
           try {
             stream.close();
           } catch (IOException e) {
-            handleIOException(e);
+            handleIOException(e,"closing stream");
           }
         }
-      }
 
+      }
+    } catch (AlfrescoDownException e) {
+      handleAlfrescoDownException(e,"processing");
     }
   }
+
+  protected final static long interruptionRetryTime = 5L*60L*1000L;
   
-  protected static void handleIOException(IOException e)
+  protected static void handleAlfrescoDownException(AlfrescoDownException e, String context)
     throws ManifoldCFException, ServiceInterruption {
-    // MHL
-    throw new ManifoldCFException(
-                                      "Error Ingesting Document:" +e.getMessage(), e);
+    long currentTime = System.currentTimeMillis();
+
+    // Server doesn't appear to by up.  Try for a brief time then give up.
+    String message = "Server appears down during "+context+": "+e.getMessage();
+    Logging.connectors.warn(message,e);
+    throw new ServiceInterruption(message,
+      e,
+      currentTime + interruptionRetryTime,
+      -1L,
+      3,
+      true);
+  }
+  
+  protected static void handleIOException(IOException e, String context)
+    throws ManifoldCFException, ServiceInterruption
+  {
+    if ((e instanceof InterruptedIOException) && (!(e instanceof java.net.SocketTimeoutException)))
+      throw new ManifoldCFException(e.getMessage(), ManifoldCFException.INTERRUPTED);
+
+    long currentTime = System.currentTimeMillis();
+    
+    if (e instanceof java.net.ConnectException)
+    {
+      // Server isn't up at all.  Try for a brief time then give up.
+      String message = "Server could not be contacted during "+context+": "+e.getMessage();
+      Logging.connectors.warn(message,e);
+      throw new ServiceInterruption(message,
+        e,
+        currentTime + interruptionRetryTime,
+        -1L,
+        3,
+        true);
+    }
+    
+    if (e instanceof java.net.SocketTimeoutException)
+    {
+      String message2 = "Socket timeout exception during "+context+": "+e.getMessage();
+      Logging.connectors.warn(message2,e);
+      throw new ServiceInterruption(message2,
+        e,
+        currentTime + interruptionRetryTime,
+        currentTime + 20L * 60000L,
+        -1,
+        false);
+    }
+      
+    if (e.getClass().getName().equals("java.net.SocketException"))
+    {
+      // In the past we would have treated this as a straight document rejection, and
+      // treated it in the same manner as a 400.  The reasoning is that the server can
+      // perfectly legally send out a 400 and drop the connection immediately thereafter,
+      // this a race condition.
+      // However, Solr 4.0 (or the Jetty version that the example runs on) seems
+      // to have a bug where it drops the connection when two simultaneous documents come
in
+      // at the same time.  This is the final version of Solr 4.0 so we need to deal with
+      // this.
+      if (e.getMessage().toLowerCase(Locale.ROOT).indexOf("broken pipe") != -1 ||
+        e.getMessage().toLowerCase(Locale.ROOT).indexOf("connection reset") != -1 ||
+        e.getMessage().toLowerCase(Locale.ROOT).indexOf("target server failed to respond")
!= -1)
+      {
+        // Treat it as a service interruption, but with a limited number of retries.
+        // In that way we won't burden the user with a huge retry interval; it should
+        // give up fairly quickly, and yet NOT give up if the error was merely transient
+        String message = "Server dropped connection during "+context+": "+e.getMessage();
+        Logging.connectors.warn(message,e);
+        throw new ServiceInterruption(message,
+          e,
+          currentTime + interruptionRetryTime,
+          -1L,
+          3,
+          false);
+      }
+      
+      // Other socket exceptions are service interruptions - but if we keep getting them,
it means 
+      // that a socket timeout is probably set too low to accept this particular document.
 So
+      // we retry for a while, then skip the document.
+      String message2 = "Socket exception during "+context+": "+e.getMessage();
+      Logging.connectors.warn(message2,e);
+      throw new ServiceInterruption(message2,
+        e,
+        currentTime + interruptionRetryTime,
+        currentTime + 20L * 60000L,
+        -1,
+        false);
+    }
 
+    // Otherwise, no idea what the trouble is, so presume that retries might fix it.
+    String message3 = "IO exception during "+context+": "+e.getMessage();
+    Logging.connectors.warn(message3,e);
+    throw new ServiceInterruption(message3,
+      e,
+      currentTime + interruptionRetryTime,
+      currentTime + 2L * 60L * 60000L,
+      -1,
+      true);
   }
   
   @Override

Modified: manifoldcf/branches/CONNECTORS-1037/connectors/alfresco-webscript/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/alfrescowebscript/ConfigurationHandler.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1037/connectors/alfresco-webscript/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/alfrescowebscript/ConfigurationHandler.java?rev=1629377&r1=1629376&r2=1629377&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-1037/connectors/alfresco-webscript/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/alfrescowebscript/ConfigurationHandler.java
(original)
+++ manifoldcf/branches/CONNECTORS-1037/connectors/alfresco-webscript/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/alfrescowebscript/ConfigurationHandler.java
Sat Oct  4 12:12:43 2014
@@ -112,19 +112,28 @@ public class ConfigurationHandler {
                                                List<String> tabsArray) throws ManifoldCFException,
IOException {
     tabsArray.add("Server");
     Map<String, Object> paramMap = new HashMap<String, Object>();
-    fillInParameters(paramMap, parameters);
+    fillInParameters(paramMap, out, parameters);
     Messages.outputResourceWithVelocity(out, locale, EDIT_CONFIG_HEADER, paramMap);
   }
 
   private static void fillInParameters(Map<String, Object> paramMap,
-                                       ConfigParams parameters) {
+                                       IHTTPOutput out, ConfigParams parameters) {
     for (Map.Entry<String, String> parameter : DEFAULT_CONFIGURATION_PARAMETERS
         .entrySet()) {
-      String paramValue = parameters.getParameter(parameter.getKey());
-      if (paramValue == null) {
-        paramValue = parameter.getValue();
+      String paramName = parameter.getKey();
+      if (paramName.endsWith("password")) {
+        String paramValue = parameters.getObfuscatedParameter(paramName);
+        if (paramValue == null) {
+          paramValue = parameter.getValue();
+        }
+        paramMap.put(paramName, out.mapPasswordToKey(paramValue));
+      } else {
+        String paramValue = parameters.getParameter(paramName);
+        if (paramValue == null) {
+          paramValue = parameter.getValue();
+        }
+        paramMap.put(paramName, paramValue);
       }
-      paramMap.put(parameter.getKey(), paramValue);
     }
   }
 
@@ -133,7 +142,7 @@ public class ConfigurationHandler {
       throws ManifoldCFException, IOException {
     Map<String, Object> paramMap = new HashMap<String, Object>();
     paramMap.put("tabName", tabName);
-    fillInParameters(paramMap, parameters);
+    fillInParameters(paramMap, out, parameters);
     Messages.outputResourceWithVelocity(out, locale, EDIT_CONFIG_SERVER, paramMap);
   }
 
@@ -143,7 +152,10 @@ public class ConfigurationHandler {
     for (String paramName : DEFAULT_CONFIGURATION_PARAMETERS.keySet()) {
       String paramValue = variableContext.getParameter(paramName);
       if (paramValue != null) {
-        parameters.setParameter(paramName, paramValue);
+        if (paramName.endsWith("password"))
+          parameters.setObfuscatedParameter(paramName,variableContext.mapKeyToPassword(paramValue));
+        else
+          parameters.setParameter(paramName, paramValue);
       }
     }
     return null;
@@ -153,7 +165,7 @@ public class ConfigurationHandler {
                                        IHTTPOutput out, Locale locale, ConfigParams parameters)
       throws ManifoldCFException, IOException {
     Map<String, Object> paramMap = new HashMap<String, Object>();
-    fillInParameters(paramMap, parameters);
+    fillInParameters(paramMap, out, parameters);
     Messages.outputResourceWithVelocity(out, locale, VIEW_CONFIG, paramMap);
   }
   

Modified: manifoldcf/branches/CONNECTORS-1037/connectors/alfresco-webscript/connector/src/test/java/org/apache/manifoldcf/crawler/connectors/alfrescowebscript/AlfrescoConnectorTest.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1037/connectors/alfresco-webscript/connector/src/test/java/org/apache/manifoldcf/crawler/connectors/alfrescowebscript/AlfrescoConnectorTest.java?rev=1629377&r1=1629376&r2=1629377&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-1037/connectors/alfresco-webscript/connector/src/test/java/org/apache/manifoldcf/crawler/connectors/alfrescowebscript/AlfrescoConnectorTest.java
(original)
+++ manifoldcf/branches/CONNECTORS-1037/connectors/alfresco-webscript/connector/src/test/java/org/apache/manifoldcf/crawler/connectors/alfrescowebscript/AlfrescoConnectorTest.java
Sat Oct  4 12:12:43 2014
@@ -18,6 +18,7 @@ package org.apache.manifoldcf.crawler.co
 
 import static org.mockito.Matchers.any;
 import static org.mockito.Matchers.anyInt;
+import static org.mockito.Matchers.anyLong;
 import static org.mockito.Matchers.anyString;
 import static org.mockito.Matchers.eq;
 import static org.mockito.Mockito.mock;
@@ -116,12 +117,24 @@ public class AlfrescoConnectorTest {
   @Test
   public void whenProcessingDocumentsNodeRefsAreUsedAsDocumentURI() throws Exception {
     TestDocument testDocument = new TestDocument();
+    testDocument.put("cm:modified","2014-10-02T16:15:25.124Z");
+    testDocument.put("size","115");
+    testDocument.put("mimetype","text/plain");
     IProcessActivity activities = mock(IProcessActivity.class);
+    when(activities.checkDocumentNeedsReindexing(anyString(),anyString()))
+      .thenReturn(true);
+    when(activities.checkLengthIndexable(anyLong()))
+      .thenReturn(true);
+    when(activities.checkMimeTypeIndexable(anyString()))
+      .thenReturn(true);
     IExistingVersions statuses = mock(IExistingVersions.class);
     
     when(client.fetchNode(anyString()))
     .thenReturn(new AlfrescoResponse(0, 0, "", "",
             Arrays.<Map<String, Object>>asList(testDocument)));
+
+    when(client.fetchMetadata(anyString()))
+      .thenReturn(testDocument);
     
 //    processDocuments(String[] documentIdentifiers, IExistingVersions statuses, Specification
spec,
 //            IProcessActivity activities, int jobMode, boolean usesDefaultAuthority)
@@ -129,13 +142,24 @@ public class AlfrescoConnectorTest {
 
     ArgumentCaptor<RepositoryDocument> rd = ArgumentCaptor.forClass(RepositoryDocument.class);
     verify(activities)
+            .checkDocumentNeedsReindexing(eq(TestDocument.uuid), eq("1412266525124"));
+    verify(activities)
+            .checkLengthIndexable(eq(115L));
+    verify(activities)
+            .checkMimeTypeIndexable(eq("text/plain"));
+    verify(activities)
             .ingestDocumentWithException(eq(TestDocument.uuid), anyString(),
-                    eq(TestDocument.uuid), rd.capture());
-
+                    eq(TestDocument.nodeRef), rd.capture());
+    
     Iterator<String> i = rd.getValue().getFields();
     while(i.hasNext()) {
       String fieldName = i.next();
       Object value1 = rd.getValue().getField(fieldName)[0];
+      Object[] values = testDocument.getRepositoryDocument().getField(fieldName);
+      if (values == null) {
+        System.out.println("field "+fieldName+"has no value in testDocument");
+        continue;
+      }
       Object value2 = testDocument.getRepositoryDocument().getField(fieldName)[0];
       assert value1.equals(value2);
     }
@@ -147,8 +171,8 @@ public class AlfrescoConnectorTest {
     testDocument.setDeleted(true);
 
     when(client.fetchNode(anyString()))
-    .thenReturn(new AlfrescoResponse(0, 0, "", "",
-            Arrays.<Map<String, Object>>asList(testDocument)));
+      .thenReturn(new AlfrescoResponse(0, 0, "", "",
+              Arrays.<Map<String, Object>>asList(testDocument)));
     
     IProcessActivity activities = mock(IProcessActivity.class);
     IExistingVersions statuses = mock(IExistingVersions.class);

Modified: manifoldcf/branches/CONNECTORS-1037/framework/core/src/main/java/org/apache/manifoldcf/core/common/DateParser.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1037/framework/core/src/main/java/org/apache/manifoldcf/core/common/DateParser.java?rev=1629377&r1=1629376&r2=1629377&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-1037/framework/core/src/main/java/org/apache/manifoldcf/core/common/DateParser.java
(original)
+++ manifoldcf/branches/CONNECTORS-1037/framework/core/src/main/java/org/apache/manifoldcf/core/common/DateParser.java
Sat Oct  4 12:12:43 2014
@@ -51,7 +51,10 @@ public class DateParser
       if (isoDateValue.indexOf(".") != -1)
         isoFormatString.append(".SSS");
       if (isoDateValue.endsWith("Z"))
-        isoFormatString.append("'Z'");
+      {
+        isoDateValue = isoDateValue.substring(0,isoDateValue.length()-1) + "-0000";
+        isoFormatString.append("Z");
+      }
       else
       {
         // We need to be able to parse either "-08:00" or "-0800".  The 'Z' specifier only
handles

Modified: manifoldcf/branches/CONNECTORS-1037/framework/core/src/test/java/org/apache/manifoldcf/core/common/DateTest.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1037/framework/core/src/test/java/org/apache/manifoldcf/core/common/DateTest.java?rev=1629377&r1=1629376&r2=1629377&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-1037/framework/core/src/test/java/org/apache/manifoldcf/core/common/DateTest.java
(original)
+++ manifoldcf/branches/CONNECTORS-1037/framework/core/src/test/java/org/apache/manifoldcf/core/common/DateTest.java
Sat Oct  4 12:12:43 2014
@@ -31,8 +31,9 @@ public class DateTest
   {
     Date d = DateParser.parseISO8601Date("96-11-15T01:32:33.344GMT");
     assertNotNull(d);
-    d = DateParser.parseISO8601Date("2012-11-15T01:32:33.344Z");
-    assertNotNull(d);
+    Date d2 = DateParser.parseISO8601Date("1996-11-15T01:32:33.344Z");
+    assertNotNull(d2);
+    assertEquals(d,d2);
     d = DateParser.parseISO8601Date("2012-11-15T01:32:33Z");
     assertNotNull(d);
     d = DateParser.parseISO8601Date("2012-11-15T01:32:33+0100");



Mime
View raw message