manifoldcf-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kwri...@apache.org
Subject svn commit: r1411041 - in /manifoldcf/branches/CONNECTORS-120/connectors/wiki/connector/src/main: java/org/apache/manifoldcf/crawler/connectors/wiki/ native2ascii/org/apache/manifoldcf/crawler/connectors/wiki/
Date Mon, 19 Nov 2012 01:46:49 GMT
Author: kwright
Date: Mon Nov 19 01:46:48 2012
New Revision: 1411041

URL: http://svn.apache.org/viewvc?rev=1411041&view=rev
Log:
Add support for wiki User-Agent header, so I can connect to wikipedia

Modified:
    manifoldcf/branches/CONNECTORS-120/connectors/wiki/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/wiki/WikiConfig.java
    manifoldcf/branches/CONNECTORS-120/connectors/wiki/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/wiki/WikiConnector.java
    manifoldcf/branches/CONNECTORS-120/connectors/wiki/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/wiki/common_en_US.properties
    manifoldcf/branches/CONNECTORS-120/connectors/wiki/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/wiki/common_ja_JP.properties

Modified: manifoldcf/branches/CONNECTORS-120/connectors/wiki/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/wiki/WikiConfig.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-120/connectors/wiki/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/wiki/WikiConfig.java?rev=1411041&r1=1411040&r2=1411041&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-120/connectors/wiki/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/wiki/WikiConfig.java
(original)
+++ manifoldcf/branches/CONNECTORS-120/connectors/wiki/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/wiki/WikiConfig.java
Mon Nov 19 01:46:48 2012
@@ -35,7 +35,11 @@ public class WikiConfig
   public static final String PARAM_PORT = "Server port";
   /** Path */
   public static final String PARAM_PATH = "Server path";
-	
+  
+  /** Email */
+  public static final String PARAM_EMAIL = "Email";
+  
+  // Login info
   public static final String PARAM_LOGIN = "serverlogin";
   public static final String PARAM_PASSWORD = "serverpass";
   public static final String PARAM_DOMAIN = "serverdomain";

Modified: manifoldcf/branches/CONNECTORS-120/connectors/wiki/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/wiki/WikiConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-120/connectors/wiki/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/wiki/WikiConnector.java?rev=1411041&r1=1411040&r2=1411041&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-120/connectors/wiki/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/wiki/WikiConnector.java
(original)
+++ manifoldcf/branches/CONNECTORS-120/connectors/wiki/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/wiki/WikiConnector.java
Mon Nov 19 01:46:48 2012
@@ -48,6 +48,7 @@ import org.apache.http.NameValuePair;
 import org.apache.http.message.BasicNameValuePair;
 import org.apache.http.protocol.HTTP;
 import org.apache.http.util.EntityUtils;
+import org.apache.http.message.BasicHeader;
 
 import org.apache.http.conn.ConnectTimeoutException;
 import org.apache.http.client.CircularRedirectException;
@@ -58,6 +59,8 @@ import java.util.*;
 import java.io.*;
 import java.net.*;
 
+import java.util.concurrent.TimeUnit;
+
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
@@ -89,6 +92,9 @@ public class WikiConnector extends org.a
   /** Base URL */
   protected String baseURL = null;
   
+  /** The user-agent for this connector instance */
+  protected String userAgent = null;
+
   protected String serverLogin = null;
   protected String serverPass = null;
   protected String serverDomain = null;
@@ -140,6 +146,12 @@ public class WikiConnector extends org.a
   {
     if (hasBeenSetup == false)
     {
+      String emailAddress = params.getParameter(WikiConfig.PARAM_EMAIL);
+      if (emailAddress != null)
+        userAgent = "Mozilla/5.0 (ApacheManifoldCFWikiReader; "+((emailAddress==null)?"":emailAddress)+")";
+      else
+        userAgent = null;
+
       String protocol = params.getParameter(WikiConfig.PARAM_PROTOCOL);
       if (protocol == null || protocol.length() == 0)
         protocol = "http";
@@ -627,9 +639,8 @@ public class WikiConnector extends org.a
   public void poll()
     throws ManifoldCFException
   {
-    // MHL
-    //if (connectionManager != null)
-    //  connectionManager.closeIdleConnections(60000L);
+    if (connectionManager != null)
+      connectionManager.closeIdleConnections(60000L,TimeUnit.MILLISECONDS);
   }
 
   /** Close the connection.  Call this before discarding the connection.
@@ -644,7 +655,8 @@ public class WikiConnector extends org.a
     serverPass = null;
     serverDomain = null;
     baseURL = null;
-    
+    userAgent = null;
+
     if (httpClient != null) {
       httpClient = null;
     }
@@ -828,12 +840,19 @@ public class WikiConnector extends org.a
     throws ManifoldCFException, IOException
   {
     tabsArray.add(Messages.getString(locale,"WikiConnector.Server"));
+    tabsArray.add(Messages.getString(locale,"WikiConnector.Email"));
 
     out.print(
 "<script type=\"text/javascript\">\n"+
 "<!--\n"+
 "function checkConfig()\n"+
 "{\n"+
+"  if (editconnection.email.value != \"\" && editconnection.email.value.indexOf(\"@\")
== -1)\n"+
+"  {\n"+
+"    alert(\""+Messages.getBodyJavascriptString(locale,"WikiConnector.NeedAValidEmailAddress")+"\");\n"+
+"    editconnection.email.focus();\n"+
+"    return false;\n"+
+"  }\n"+
 "  if (editconnection.serverport.value != \"\" && !isInteger(editconnection.serverport.value))\n"+
 "  {\n"+
 "    alert(\""+Messages.getBodyJavascriptString(locale,"WikiConnector.WikiServerPortMustBeAValidInteger")+"\");\n"+
@@ -851,6 +870,13 @@ public class WikiConnector extends org.a
 "\n"+
 "function checkConfigForSave()\n"+
 "{\n"+
+"  if (editconnection.email.value == \"\")\n"+
+"  {\n"+
+"    alert(\""+Messages.getBodyJavascriptString(locale,"WikiConnector.EmailAddressRequiredToBeIncludedInAllRequestHeaders")+"\");\n"+
+"    SelectTab(\""+Messages.getBodyJavascriptString(locale,"WikiConnector.Email")+"\");\n"+
+"    editconnection.email.focus();\n"+
+"    return false;\n"+
+"  }\n"+
 "  if (editconnection.servername.value == \"\")\n"+
 "  {\n"+
 "    alert(\""+Messages.getBodyJavascriptString(locale,"WikiConnector.PleaseSupplyAValidWikiServerName")+"\");\n"+
@@ -893,6 +919,10 @@ public class WikiConnector extends org.a
     Locale locale, ConfigParams parameters, String tabName)
     throws ManifoldCFException, IOException
   {
+    String email = parameters.getParameter(WikiConfig.PARAM_EMAIL);
+    if (email == null)
+      email = "";
+
     String protocol = parameters.getParameter(WikiConfig.PARAM_PROTOCOL);
     if (protocol == null)
       protocol = "http";
@@ -922,6 +952,25 @@ public class WikiConnector extends org.a
       domain = "";
     }
     
+    // Email tab
+    if (tabName.equals(Messages.getString(locale,"RSSConnector.Email")))
+    {
+      out.print(
+"<table class=\"displaytable\">\n"+
+"  <tr><td class=\"separator\" colspan=\"2\"><hr/></td></tr>\n"+
+"  <tr>\n"+
+"    <td class=\"description\"><nobr>" + Messages.getBodyString(locale,"WikiConnector.EmailAddressToContactColon")
+ "</nobr></td><td class=\"value\"><input type=\"text\" size=\"32\" name=\"email\"
value=\""+org.apache.manifoldcf.ui.util.Encoder.attributeEscape(email)+"\"/></td>\n"+
+"  </tr>\n"+
+"</table>\n"
+      );
+    }
+    else
+    {
+      out.print(
+"<input type=\"hidden\" name=\"email\" value=\""+org.apache.manifoldcf.ui.util.Encoder.attributeEscape(email)+"\"/>\n"
+      );
+    }
+
     if (tabName.equals(Messages.getString(locale,"WikiConnector.Server")))
     {
       out.print(
@@ -1004,6 +1053,10 @@ public class WikiConnector extends org.a
     Locale locale, ConfigParams parameters)
     throws ManifoldCFException
   {
+    String email = variableContext.getParameter("email");
+    if (email != null)
+      parameters.setParameter(WikiConfig.PARAM_EMAIL,email);
+
     String protocol = variableContext.getParameter("serverprotocol");
     if (protocol != null)
       parameters.setParameter(WikiConfig.PARAM_PROTOCOL,protocol);
@@ -1613,7 +1666,10 @@ public class WikiConnector extends org.a
   protected HttpRequestBase getInitializedGetMethod(String URL)
     throws IOException
   {
-    return new HttpGet(URL);
+    HttpGet method = new HttpGet(URL);
+    if (userAgent != null)
+      method.setHeader(new BasicHeader("User-Agent",userAgent));
+    return method;
   }
 
   /** Create an initialize a post method */
@@ -1621,6 +1677,9 @@ public class WikiConnector extends org.a
     throws IOException
   {
     HttpPost method = new HttpPost(URL);
+    if (userAgent != null)
+      method.setHeader(new BasicHeader("User-Agent",userAgent));
+
     List<NameValuePair> pairs = new ArrayList<NameValuePair>();
     
     for (String key : params.keySet()) {

Modified: manifoldcf/branches/CONNECTORS-120/connectors/wiki/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/wiki/common_en_US.properties
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-120/connectors/wiki/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/wiki/common_en_US.properties?rev=1411041&r1=1411040&r2=1411041&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-120/connectors/wiki/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/wiki/common_en_US.properties
(original)
+++ manifoldcf/branches/CONNECTORS-120/connectors/wiki/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/wiki/common_en_US.properties
Mon Nov 19 01:46:48 2012
@@ -23,6 +23,7 @@ WikiConnector.ServerPassword=API passwor
 WikiConnector.ServerDomain=API domain:
 WikiConnector.NamespaceAndTitles=Namespace and Titles
 WikiConnector.NamespaceAndTitles2=Namespaces and titles:
+WikiConnector.Email=Email
 WikiConnector.Namespace=Namespace
 WikiConnector.TitlePrefix=Title prefix
 WikiConnector.Security=Security
@@ -40,6 +41,9 @@ WikiConnector.AllDefaultNamespaceDocumen
 WikiConnector.WikiServerPortMustBeAValidInteger=Wiki server port must be a valid integer
 WikiConnector.PathMustStartWithACharacter=Path must start with a '/' character
 WikiConnector.PleaseSupplyAValidWikiServerName=Please supply a valid Wiki server name
+WikiConnector.NeedAValidEmailAddress=Need a valid email address
+WikiConnector.EmailAddressRequiredToBeIncludedInAllRequestHeaders=Email address required
to be included in all request headers
+WikiConnector.EmailAddressToContactColon=Email address to contact:
 WikiConnector.Parameters=Parameters:
 WikiConnector.certificates= certificate(s)
 WikiConnector.DeleteNamespaceTitle=Delete namespace/title #

Modified: manifoldcf/branches/CONNECTORS-120/connectors/wiki/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/wiki/common_ja_JP.properties
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-120/connectors/wiki/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/wiki/common_ja_JP.properties?rev=1411041&r1=1411040&r2=1411041&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-120/connectors/wiki/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/wiki/common_ja_JP.properties
(original)
+++ manifoldcf/branches/CONNECTORS-120/connectors/wiki/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/wiki/common_ja_JP.properties
Mon Nov 19 01:46:48 2012
@@ -23,6 +23,7 @@ WikiConnector.ServerPassword=API passwor
 WikiConnector.ServerDomain=API domain:
 WikiConnector.NamespaceAndTitles=名前空間と題名
 WikiConnector.NamespaceAndTitles2=名前空間と題名:
+WikiConnector.Email=Email
 WikiConnector.Namespace=名前空間ん
 WikiConnector.TitlePrefix=題名接先頭辞
 WikiConnector.Security=Security
@@ -36,10 +37,13 @@ WikiConnector.Add=追加
 WikiConnector.AddNamespacePrefix=名前空間/接先頭辞の追加
 WikiConnector.UseDefault=デフォルトを使う
 WikiConnector.TransientError=一時的エラー: 
-WikiConnector.=AllDefaultNamespaceDocumentsIncluded=すべてのデフォルト名前空間を含む
+WikiConnector.AllDefaultNamespaceDocumentsIncluded=すべてのデフォルト名前空間を含む
 WikiConnector.WikiServerPortMustBeAValidInteger=Wikiサーバポートには整数を入力してください
 WikiConnector.PathMustStartWithACharacter=パス名の先頭は「/」にしてください
 WikiConnector.PleaseSupplyAValidWikiServerName=正しいWikiサーバ名を入力してください
+WikiConnector.NeedAValidEmailAddress=Need a valid email address
+WikiConnector.EmailAddressRequiredToBeIncludedInAllRequestHeaders=Email address required
to be included in all request headers
+WikiConnector.EmailAddressToContactColon=Email address to contact:
 WikiConnector.Parameters=引数:
 WikiConnector.certificates= 証明証
 WikiConnector.DeleteNamespaceTitle=名前空間/題名を削除:
#



Mime
View raw message