incubator-connectors-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kwri...@apache.org
Subject svn commit: r1171158 - in /incubator/lcf/trunk: ./ connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/
Date Thu, 15 Sep 2011 16:07:32 GMT
Author: kwright
Date: Thu Sep 15 16:07:32 2011
New Revision: 1171158

URL: http://svn.apache.org/viewvc?rev=1171158&view=rev
Log:
Fix for CONNECTORS-202.  Add ability to set the commit within parameter for Solr.

Modified:
    incubator/lcf/trunk/CHANGES.txt
    incubator/lcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java
    incubator/lcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConfig.java
    incubator/lcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConnector.java

Modified: incubator/lcf/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/CHANGES.txt?rev=1171158&r1=1171157&r2=1171158&view=diff
==============================================================================
--- incubator/lcf/trunk/CHANGES.txt (original)
+++ incubator/lcf/trunk/CHANGES.txt Thu Sep 15 16:07:32 2011
@@ -3,6 +3,9 @@ $Id$
 
 ======================= 0.4-dev =====================
 
+CONNECTORS-202: Add commit-within parameter to Solr output connector.
+(Jan H√łydahl, Karl Wright)
+
 CONNECTORS-251: Add target "download-dependencies" to build.xml.
 (Shinichiro Abe)
 

Modified: incubator/lcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java?rev=1171158&r1=1171157&r2=1171158&view=diff
==============================================================================
--- incubator/lcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java
(original)
+++ incubator/lcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java
Thu Sep 15 16:07:32 2011
@@ -70,9 +70,12 @@ public class HttpPoster
   
   private Long maxDocumentLength;
 
+  private String commitWithin;
+  
   private static final String LITERAL = "literal.";
   private static final String NOTHING = "__NOTHING__";
   private static final String ID_METADATA = "lcf_metadata_id";
+  private static final String COMMITWITHIN_METADATA = "commitWithin";
   
   private int buffersize = 32768;  // default buffer size
   double sizeCoefficient = 0.0005;    // 20 ms additional timeout per 2000 bytes, pulled
out of my butt
@@ -131,9 +134,12 @@ public class HttpPoster
     String updatePath, String removePath, String statusPath,
     String realm, String userID, String password,
     String allowAttributeName, String denyAttributeName, String idAttributeName,
-    IKeystoreManager keystoreManager, Long maxDocumentLength)
+    IKeystoreManager keystoreManager, Long maxDocumentLength,
+    String commitWithin)
     throws ManifoldCFException
   {
+    this.commitWithin = commitWithin;
+    
     this.allowAttributeName = allowAttributeName;
     this.denyAttributeName = denyAttributeName;
     this.idAttributeName = idAttributeName;
@@ -300,7 +306,7 @@ public class HttpPoster
     {
       try
       {
-        IngestThread t = new IngestThread(documentURI,document,arguments,sourceTargets,shareAcls,shareDenyAcls,acls,denyAcls);
+        IngestThread t = new IngestThread(documentURI,document,arguments,sourceTargets,shareAcls,shareDenyAcls,acls,denyAcls,commitWithin);
         try
         {
           t.start();
@@ -1024,7 +1030,8 @@ public class HttpPoster
     protected String[] shareDenyAcls;
     protected String[] acls;
     protected String[] denyAcls;
-
+    protected String commitWithin;
+    
     protected Long activityStart = null;
     protected Long activityBytes = null;
     protected String activityCode = null;
@@ -1034,7 +1041,7 @@ public class HttpPoster
     protected boolean rval = false;
 
     public IngestThread(String documentURI, RepositoryDocument document, Map arguments, Map
sourceTargets,
-      String[] shareAcls, String[] shareDenyAcls, String[] acls, String[] denyAcls)
+      String[] shareAcls, String[] shareDenyAcls, String[] acls, String[] denyAcls, String
commitWithin)
     {
       super();
       setDaemon(true);
@@ -1046,6 +1053,7 @@ public class HttpPoster
       this.acls = acls;
       this.denyAcls = denyAcls;
       this.sourceTargets = sourceTargets;
+      this.commitWithin = commitWithin;
     }
 
     public void run()
@@ -1134,6 +1142,9 @@ public class HttpPoster
                     }
                   }
                 }
+                // Count the commitWithin parameter
+                if (commitWithin != null)
+                  totalLength += lengthField(COMMITWITHIN_METADATA,commitWithin);
                 // Count the binary data
                 totalLength += lengthPreamble();
                 totalLength += lengthBoundary("application/octet-stream","myfile","docname");
@@ -1198,6 +1209,10 @@ public class HttpPoster
                     }
                   }
                 }
+                
+                // Write the commitWithin parameter
+                if (commitWithin != null)
+                  writeField(out,COMMITWITHIN_METADATA,commitWithin);
 
                 // Write the content
                 writePreamble(out);

Modified: incubator/lcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConfig.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConfig.java?rev=1171158&r1=1171157&r2=1171158&view=diff
==============================================================================
--- incubator/lcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConfig.java
(original)
+++ incubator/lcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConfig.java
Thu Sep 15 16:07:32 2011
@@ -53,6 +53,8 @@ public class SolrConfig
   public static final String PARAM_PASSWORD = "Password";
   /** Enable commits */
   public static final String PARAM_COMMITS = "Commits";
+  /** Commit within time */
+  public static final String PARAM_COMMITWITHIN = "Commit within";
   /** Keystore */
   public static final String PARAM_KEYSTORE = "Keystore";
   /** Maximum document length */

Modified: incubator/lcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConnector.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConnector.java?rev=1171158&r1=1171157&r2=1171158&view=diff
==============================================================================
--- incubator/lcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConnector.java
(original)
+++ incubator/lcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConnector.java
Thu Sep 15 16:07:32 2011
@@ -146,6 +146,10 @@ public class SolrConnector extends org.a
       
       doCommits = commits.equals("true");
       
+      String commitWithin = params.getParameter(SolrConfig.PARAM_COMMITWITHIN);
+      if (commitWithin == null || commitWithin.length() == 0)
+        commitWithin = null;
+      
       String docMax = params.getParameter(SolrConfig.PARAM_MAXLENGTH);
       if (docMax == null || docMax.length() == 0)
         maxDocumentLength = null;
@@ -206,7 +210,7 @@ public class SolrConnector extends org.a
       try
       {
         poster = new HttpPoster(protocol,server,Integer.parseInt(port),webapp,updatePath,removePath,statusPath,realm,userID,password,
-          allowAttributeName,denyAttributeName,idAttributeName,keystoreManager,maxDocumentLength);
+          allowAttributeName,denyAttributeName,idAttributeName,keystoreManager,maxDocumentLength,commitWithin);
       }
       catch (NumberFormatException e)
       {
@@ -649,6 +653,12 @@ public class SolrConnector extends org.a
 "    editconnection.maxdocumentlength.focus();\n"+
 "    return false;\n"+
 "  }\n"+
+"  if (editconnection.commitwithin.value != \"\" && !isInteger(editconnection.commitwithin.value))\n"+
+"  {\n"+
+"    alert(\"Commit-within value must be an integer\");\n"+
+"    editconnection.commitwithin.focus();\n"+
+"    return false;\n"+
+"  }\n"+
 "  return true;\n"+
 "}\n"+
 "\n"+
@@ -717,6 +727,13 @@ public class SolrConnector extends org.a
 "    editconnection.maxdocumentlength.focus();\n"+
 "    return false;\n"+
 "  }\n"+
+"  if (editconnection.commitwithin.value != \"\" && !isInteger(editconnection.commitwithin.value))\n"+
+"  {\n"+
+"    alert(\"Commit-within value must be an integer\");\n"+
+"    SelectTab(\"Commits\");\n"+
+"    editconnection.commitwithin.focus();\n"+
+"    return false;\n"+
+"  }\n"+
 "  return true;\n"+
 "}\n"+
 "\n"+
@@ -816,6 +833,10 @@ public class SolrConnector extends org.a
     if (commits == null)
       commits = "true";
     
+    String commitWithin = parameters.getParameter(SolrConfig.PARAM_COMMITWITHIN);
+    if (commitWithin == null)
+      commitWithin = "";
+
     String solrKeystore = parameters.getParameter(SolrConfig.PARAM_KEYSTORE);
     IKeystoreManager localKeystore;
     if (solrKeystore == null)
@@ -1050,6 +1071,12 @@ public class SolrConnector extends org.a
 "      <input name=\"commits\" type=\"checkbox\" value=\"true\""+(commits.equals("true")?"
checked=\"yes\"":"")+"/>\n"+
 "    </td>\n"+
 "  </tr>\n"+
+"  <tr>\n"+
+"    <td class=\"description\"><nobr>Commit each document within (ms):</nobr></td>\n"+
+"    <td class=\"value\">\n"+
+"      <input name=\"commitwithin\" type=\"text\" size=\"16\" value=\""+commitWithin+"\"/>\n"+
+"    </td>\n"+
+"  </tr>\n"+
 "</table>\n"
       );
     }
@@ -1057,7 +1084,8 @@ public class SolrConnector extends org.a
     {
       out.print(
 "<input type=\"hidden\" name=\"commits_present\" value=\"true\"/>\n"+
-"<input name=\"commits\" type=\"hidden\" value=\""+commits+"\"/>\n"
+"<input name=\"commits\" type=\"hidden\" value=\""+commits+"\"/>\n"+
+"<input name=\"commitwithin\" type=\"hidden\" value=\""+commitWithin+"\"/>\n"
       );
     }
 
@@ -1274,6 +1302,10 @@ public class SolrConnector extends org.a
       parameters.setParameter(SolrConfig.PARAM_COMMITS,commits);
     }
     
+    String commitWithin = variableContext.getParameter("commitwithin");
+    if (commitWithin != null)
+      parameters.setParameter(SolrConfig.PARAM_COMMITWITHIN,commitWithin);
+    
     String keystoreValue = variableContext.getParameter("keystoredata");
     IKeystoreManager mgr;
     if (keystoreValue != null)



Mime
View raw message